From 6e023f7cf2f63b009408584bf16e3efcad73254f Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:45:57 -0700 Subject: [PATCH 01/18] fix(model_prices): apply claude-sonnet-5 introductory pricing through 2026-08-31 (#31917) * fix(model_prices): apply claude-sonnet-5 introductory pricing through 2026-08-31 Anthropic launched Sonnet 5 with introductory pricing of $2/$10 per million input/output tokens through August 31, 2026 (sticker price $3/$15 applies from September 1, 2026). Bedrock, Vertex AI, and Azure Foundry mirror the introductory rate. LiteLLM was charging the sticker price on all ten claude-sonnet-5 entries, over-billing by 50% during the introductory period. Update input, output, cache write (5m and 1h), and cache read costs on the base entries to the introductory rate, and keep the 10% cross-region premium on the us/eu/au/jp Bedrock inference profiles on top of it. Also add an anthropic-sonnet-5 entry to the dev proxy config. * test: document exact sticker prices to restore on 2026-09-01 --- ...odel_prices_and_context_window_backup.json | 100 +++++++++--------- litellm/proxy/dev_config.yaml | 4 + model_prices_and_context_window.json | 100 +++++++++--------- .../test_claude_sonnet_5_config.py | 41 ++++--- 4 files changed, 131 insertions(+), 114 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b3058bb4214..b249b951b8e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1672,16 +1672,16 @@ "supports_minimal_reasoning_effort": true }, "anthropic.claude-sonnet-5": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -1705,16 +1705,16 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "global.anthropic.claude-sonnet-5": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -1738,16 +1738,16 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-sonnet-5": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_creation_input_token_cost_above_1hr": 6.6e-06, - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -1771,16 +1771,16 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-sonnet-5": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_creation_input_token_cost_above_1hr": 6.6e-06, - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -1804,16 +1804,16 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-sonnet-5": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_creation_input_token_cost_above_1hr": 6.6e-06, - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -1837,16 +1837,16 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "jp.anthropic.claude-sonnet-5": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_creation_input_token_cost_above_1hr": 6.6e-06, - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -2710,16 +2710,16 @@ "supports_vision": true }, "azure_ai/claude-sonnet-5": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -10474,16 +10474,16 @@ "supports_web_search": true }, "claude-sonnet-5": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -35210,16 +35210,16 @@ "supports_vision": true }, "vertex_ai/claude-sonnet-5": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -42677,16 +42677,16 @@ } }, "vertex_ai/claude-sonnet-5@default": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, diff --git a/litellm/proxy/dev_config.yaml b/litellm/proxy/dev_config.yaml index 6078161c780..0dc90e658f2 100644 --- a/litellm/proxy/dev_config.yaml +++ b/litellm/proxy/dev_config.yaml @@ -28,6 +28,10 @@ model_list: litellm_params: model: anthropic/claude-opus-4-8 api_key: os.environ/ANTHROPIC_API_KEY + - model_name: anthropic-sonnet-5 + litellm_params: + model: anthropic/claude-sonnet-5 + api_key: os.environ/ANTHROPIC_API_KEY # ---------- Bedrock Invoke ---------- - model_name: bedrock-invoke-haiku-4-5 diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6ab6f1bda46..9e9523193c7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1672,16 +1672,16 @@ "supports_minimal_reasoning_effort": true }, "anthropic.claude-sonnet-5": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -1705,16 +1705,16 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "global.anthropic.claude-sonnet-5": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -1738,16 +1738,16 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-sonnet-5": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_creation_input_token_cost_above_1hr": 6.6e-06, - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -1771,16 +1771,16 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-sonnet-5": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_creation_input_token_cost_above_1hr": 6.6e-06, - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -1804,16 +1804,16 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-sonnet-5": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_creation_input_token_cost_above_1hr": 6.6e-06, - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -1837,16 +1837,16 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "jp.anthropic.claude-sonnet-5": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_creation_input_token_cost_above_1hr": 6.6e-06, - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -2710,16 +2710,16 @@ "supports_vision": true }, "azure_ai/claude-sonnet-5": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -10474,16 +10474,16 @@ "supports_web_search": true }, "claude-sonnet-5": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -35384,16 +35384,16 @@ "supports_vision": true }, "vertex_ai/claude-sonnet-5": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -42909,16 +42909,16 @@ } }, "vertex_ai/claude-sonnet-5@default": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, diff --git a/tests/test_litellm/test_claude_sonnet_5_config.py b/tests/test_litellm/test_claude_sonnet_5_config.py index 88a06aea95f..506ffa16597 100644 --- a/tests/test_litellm/test_claude_sonnet_5_config.py +++ b/tests/test_litellm/test_claude_sonnet_5_config.py @@ -76,12 +76,23 @@ def test_sonnet_5_pricing_and_capabilities(): assert info["max_output_tokens"] == 128000 assert info["max_tokens"] == 128000 - # Standard Sonnet pricing: $3 / $15 per MTok, with the 1.25x cache-write - # and 0.1x cache-read multipliers. - assert info["input_cost_per_token"] == 3e-06 - assert info["output_cost_per_token"] == 1.5e-05 - assert info["cache_creation_input_token_cost"] == 3.75e-06 - assert info["cache_read_input_token_cost"] == 3e-07 + # Introductory Sonnet 5 pricing through 2026-08-31: $2 / $10 per MTok, + # with the 1.25x cache-write and 0.1x cache-read multipliers. On + # 2026-09-01 flip these five fields back to the sticker rate, here and + # in both cost-map JSON files (all ten claude-sonnet-5 entries): + # input_cost_per_token: 3e-06 + # output_cost_per_token: 1.5e-05 + # cache_creation_input_token_cost: 3.75e-06 + # cache_creation_input_token_cost_above_1hr: 6e-06 + # cache_read_input_token_cost: 3e-07 + # Regional Bedrock profiles (us./eu./au./jp.) stay at 1.1x those values: + # 3.3e-06 / 1.65e-05 / 4.125e-06 / 6.6e-06 / 3.3e-07 (see + # test_sonnet_5_bedrock_regional_pricing below). + assert info["input_cost_per_token"] == 2e-06 + assert info["output_cost_per_token"] == 1e-05 + assert info["cache_creation_input_token_cost"] == 2.5e-06 + assert info["cache_creation_input_token_cost_above_1hr"] == 4e-06 + assert info["cache_read_input_token_cost"] == 2e-07 # gen-5 adaptive-thinking profile: effort-driven, no sampling params, no # assistant prefill. @@ -102,16 +113,18 @@ def test_sonnet_5_bedrock_regional_pricing(): model_data = _load_root_cost_map() base_pricing = { - "input_cost_per_token": 3e-06, - "output_cost_per_token": 1.5e-05, - "cache_creation_input_token_cost": 3.75e-06, - "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, } regional_pricing = { - "input_cost_per_token": 3.3e-06, - "output_cost_per_token": 1.65e-05, - "cache_creation_input_token_cost": 4.125e-06, - "cache_read_input_token_cost": 3.3e-07, + "input_cost_per_token": 2.2e-06, + "output_cost_per_token": 1.1e-05, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, } expected = { From 8ce6b4d712dcc0486a8b23fdb0ead55097f5bfc1 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 1 Jul 2026 17:58:31 -0700 Subject: [PATCH 02/18] fix(proxy): tighten role gating on /get/config/callbacks response (#31745) The handler returned decrypted callback environment values and alerting routing values verbatim to callers who were not full PROXY_ADMIN. Gate those on full-admin role, matching the posture used on the sibling config-inspection endpoints. Non-sensitive routing fields (host / base URL / port style values) stay visible so the UI can still label which integration is wired up. Full PROXY_ADMIN sees everything unchanged so the edit form round-trips on save. Resolves LIT-4115. --- litellm/proxy/common_utils/callback_utils.py | 17 +- litellm/proxy/proxy_server.py | 64 +++++- tests/proxy_unit_tests/test_proxy_server.py | 12 +- .../proxy/proxy_server/test_routes_config.py | 216 ++++++++++++++++++ tests/test_litellm/proxy/test_proxy_server.py | 20 +- .../test_router_retry_policy_update.py | 8 +- 6 files changed, 315 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 573763d6627..c644ecc3dae 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -611,10 +611,17 @@ def _transform_callback_vars(metadata: Any, transform: Callable[[str, Any], Any] return out -def _is_sensitive_callback_var(key: str) -> bool: - """Match codebase precedent: only credential-bearing fields get encrypted; - routing/identifier fields (host, base_url, project, region) stay plain.""" - if key in _EXTRA_SENSITIVE_CALLBACK_KEYS: +def is_sensitive_callback_key( + key: str, + extra: Optional[set[str]] = None, +) -> bool: + """Return ``True`` if ``key`` is present in ``extra`` (checked as-is), or + if its lowercase form is in ``_EXTRA_SENSITIVE_CALLBACK_KEYS``, or if + ``_CALLBACK_VAR_MASKER.is_sensitive_key`` matches it. + """ + if extra and key in extra: + return True + if key.lower() in _EXTRA_SENSITIVE_CALLBACK_KEYS: return True return _CALLBACK_VAR_MASKER.is_sensitive_key(key) @@ -622,7 +629,7 @@ def _is_sensitive_callback_var(key: str) -> bool: def _encrypt_if_plaintext(key: str, value: Any) -> Any: if not isinstance(value, str) or not value: return value - if not _is_sensitive_callback_var(key): + if not is_sensitive_callback_key(key): return value if value.startswith(_CALLBACK_VAR_ENCRYPTED_PREFIX): # Already encrypted — round-tripping ciphertext (e.g. UI Edit Settings diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c85faeba1a6..9a338712187 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -102,6 +102,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec from litellm.proxy.common_utils.callback_utils import ( + is_sensitive_callback_key, normalize_callback_names, process_callback, ) @@ -14310,6 +14311,50 @@ async def create_config_audit_log( ) +_EXTRA_SECRET_CALLBACK_ENV_VARS = frozenset( + { + "GALILEO_USERNAME", + "GENERIC_LOGGER_HEADERS", + "OTEL_HEADERS", + "SLACK_WEBHOOK_URL", + "SMTP_USERNAME", + } +) + + +def _redact_callback_env_vars(env_vars: dict[str, Optional[str]]) -> dict[str, Optional[str]]: + """Return a copy of ``env_vars`` with values for keys classified as + sensitive by ``is_sensitive_callback_key`` replaced with ``"REDACTED"``. + ``None`` values pass through unchanged. + """ + return { + key: ( + "REDACTED" + if value is not None and is_sensitive_callback_key(key, extra=_EXTRA_SECRET_CALLBACK_ENV_VARS) + else value + ) + for key, value in env_vars.items() + } + + +def _apply_callback_role_gate(entries: list, is_full_admin: bool) -> list: + if is_full_admin: + return entries + return [{**entry, "variables": _redact_callback_env_vars(entry.get("variables") or {})} for entry in entries] + + +def _apply_alerting_env_role_gate(env_vars: dict, is_full_admin: bool) -> dict: + if is_full_admin: + return mask_sensitive_keys(env_vars, _ALERTING_SENSITIVE_VARS) + return _redact_callback_env_vars(env_vars) + + +def _apply_webhook_role_gate(webhook_map, is_full_admin: bool): + if is_full_admin or not isinstance(webhook_map, dict): + return webhook_map + return {alert_type: "REDACTED" for alert_type in webhook_map} + + @router.get( "/config/field/info", tags=["config.yaml"], @@ -14720,7 +14765,9 @@ async def delete_callback( include_in_schema=False, dependencies=[Depends(user_api_key_auth)], ) -async def get_config(): +async def get_config( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ For Admin UI - allows admin to view config via UI # return the callbacks and the env variables for the callback @@ -14735,6 +14782,8 @@ async def get_config(): _general_settings = config_data.get("general_settings", {}) environment_variables = config_data.get("environment_variables", {}) + is_full_admin = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN + _success_callbacks = _litellm_settings.get("success_callback", []) _failure_callbacks = _litellm_settings.get("failure_callback", []) _success_and_failure_callbacks = _litellm_settings.get("callbacks", []) @@ -14776,6 +14825,8 @@ async def get_config(): for _callback in _success_and_failure_callbacks: _data_to_return.append(process_callback(_callback, "success_and_failure", environment_variables)) + _data_to_return = _apply_callback_role_gate(_data_to_return, is_full_admin) + # Check if slack alerting is on _alerting = _general_settings.get("alerting", []) alerting_data = [] @@ -14787,11 +14838,13 @@ async def get_config(): _var: (value if (value := environment_variables.get(_var)) is not None else os.getenv(_var)) for _var in _slack_vars } - _slack_env_vars = mask_sensitive_keys(_slack_env_vars, _ALERTING_SENSITIVE_VARS) + _slack_env_vars = _apply_alerting_env_role_gate(_slack_env_vars, is_full_admin) _alerting_types = proxy_logging_obj.slack_alerting_instance.alert_types _all_alert_types = proxy_logging_obj.slack_alerting_instance._all_possible_alert_types() - _alerts_to_webhook = proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url + _alerts_to_webhook = _apply_webhook_role_gate( + proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url, is_full_admin + ) alerting_data.append( { "name": "slack", @@ -14811,8 +14864,9 @@ async def get_config(): "EMAIL_LOGO_URL", "EMAIL_SUPPORT_CONTACT", ] - _email_env_vars = {_var: environment_variables.get(_var) for _var in _email_vars} - _email_env_vars = mask_sensitive_keys(_email_env_vars, _ALERTING_SENSITIVE_VARS) + _email_env_vars = _apply_alerting_env_role_gate( + {_var: environment_variables.get(_var) for _var in _email_vars}, is_full_admin + ) alerting_data.append( { diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index 921fbfa320f..212f7772cad 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -2844,7 +2844,9 @@ async def test_get_config_callbacks_with_all_types(client_no_auth): async def test_get_config_callbacks_environment_variables(client_no_auth): """ Test that /get/config/callbacks correctly includes environment variables - for each callback type. Values are returned as-is from the config (no decryption). + for each callback type. Under ``client_no_auth`` the resolved role is + not ``PROXY_ADMIN``, so values matched by the redaction helper come back + as ``"REDACTED"`` and other values pass through verbatim. """ from litellm.proxy.proxy_server import ProxyConfig @@ -2886,12 +2888,11 @@ async def test_get_config_callbacks_environment_variables(client_no_auth): assert langfuse_callback["type"] == "success" assert "variables" in langfuse_callback - # Verify langfuse env vars are present (values returned as-is, no decryption) langfuse_vars = langfuse_callback["variables"] assert "LANGFUSE_PUBLIC_KEY" in langfuse_vars - assert langfuse_vars["LANGFUSE_PUBLIC_KEY"] == "test-public-key" + assert langfuse_vars["LANGFUSE_PUBLIC_KEY"] == "REDACTED" assert "LANGFUSE_SECRET_KEY" in langfuse_vars - assert langfuse_vars["LANGFUSE_SECRET_KEY"] == "test-secret-key" + assert langfuse_vars["LANGFUSE_SECRET_KEY"] == "REDACTED" assert "LANGFUSE_HOST" in langfuse_vars assert langfuse_vars["LANGFUSE_HOST"] == "https://cloud.langfuse.com" @@ -2901,14 +2902,13 @@ async def test_get_config_callbacks_environment_variables(client_no_auth): assert otel_callback["type"] == "success_and_failure" assert "variables" in otel_callback - # Verify otel env vars are present otel_vars = otel_callback["variables"] assert "OTEL_EXPORTER" in otel_vars assert otel_vars["OTEL_EXPORTER"] == "otlp" assert "OTEL_ENDPOINT" in otel_vars assert otel_vars["OTEL_ENDPOINT"] == "http://localhost:4317" assert "OTEL_HEADERS" in otel_vars - assert otel_vars["OTEL_HEADERS"] == "key=value" + assert otel_vars["OTEL_HEADERS"] == "REDACTED" @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index df14dc5b5dc..4ac6fc46a61 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -741,6 +741,222 @@ def test_get_config_callbacks_internal_error(client, auth_as, mock_prisma, monke ) +_CALLBACK_ENV_FIXTURE = { + "LANGFUSE_PUBLIC_KEY": "pk-public-1234567890", + "LANGFUSE_SECRET_KEY": "sk-langfuse-super-secret", + "LANGFUSE_HOST": "https://cloud.langfuse.com", + "DD_API_KEY": "dd-super-secret-api-key", + "DD_SITE": "datadoghq.com", + "OTEL_HEADERS": "Authorization=Bearer otel-super-secret", + "OTEL_ENDPOINT": "https://otlp.example.com", + "SLACK_WEBHOOK_URL": "https://hooks.slack.com/services/T000/B000/SLACK-WEBHOOK-FIXTURE-SECRET", +} + + +def _install_callbacks_config(monkeypatch, mock_prisma): + from litellm.proxy import proxy_server as ps + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps, "llm_router", None) + + fake_proxy_config = MagicMock() + fake_proxy_config.get_config = AsyncMock( + return_value={ + "litellm_settings": {"success_callback": ["langfuse", "datadog", "otel"]}, + "general_settings": {"alerting": ["slack"]}, + "environment_variables": dict(_CALLBACK_ENV_FIXTURE), + } + ) + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + +def _callback_variables(body: dict, name: str) -> dict: + return next( + cb["variables"] for cb in body["callbacks"] if cb["name"] == name + ) + + +def test_get_config_callbacks_redacts_secret_env_vars_for_view_only_admin( + client, auth_as, mock_prisma, monkeypatch +): + from litellm.proxy._types import LitellmUserRoles + + _install_callbacks_config(monkeypatch, mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): + response = client.get("/get/config/callbacks") + assert response.status_code == 200 + body = response.json() + + for secret in ( + _CALLBACK_ENV_FIXTURE["LANGFUSE_SECRET_KEY"], + _CALLBACK_ENV_FIXTURE["DD_API_KEY"], + _CALLBACK_ENV_FIXTURE["OTEL_HEADERS"], + _CALLBACK_ENV_FIXTURE["LANGFUSE_PUBLIC_KEY"], + ): + assert secret not in response.text + + langfuse_vars = _callback_variables(body, "langfuse") + assert langfuse_vars["LANGFUSE_PUBLIC_KEY"] == "REDACTED" + assert langfuse_vars["LANGFUSE_SECRET_KEY"] == "REDACTED" + assert langfuse_vars["LANGFUSE_HOST"] == _CALLBACK_ENV_FIXTURE["LANGFUSE_HOST"] + + datadog_vars = _callback_variables(body, "datadog") + assert datadog_vars["DD_API_KEY"] == "REDACTED" + assert datadog_vars["DD_SITE"] == _CALLBACK_ENV_FIXTURE["DD_SITE"] + + otel_vars = _callback_variables(body, "otel") + assert otel_vars["OTEL_HEADERS"] == "REDACTED" + assert otel_vars["OTEL_ENDPOINT"] == _CALLBACK_ENV_FIXTURE["OTEL_ENDPOINT"] + + +def test_get_config_callbacks_full_admin_still_sees_secret_env_vars( + client, auth_as, mock_prisma, monkeypatch +): + from litellm.proxy._types import LitellmUserRoles + + _install_callbacks_config(monkeypatch, mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/get/config/callbacks") + assert response.status_code == 200 + body = response.json() + + langfuse_vars = _callback_variables(body, "langfuse") + assert langfuse_vars["LANGFUSE_SECRET_KEY"] == _CALLBACK_ENV_FIXTURE["LANGFUSE_SECRET_KEY"] + assert langfuse_vars["LANGFUSE_PUBLIC_KEY"] == _CALLBACK_ENV_FIXTURE["LANGFUSE_PUBLIC_KEY"] + + datadog_vars = _callback_variables(body, "datadog") + assert datadog_vars["DD_API_KEY"] == _CALLBACK_ENV_FIXTURE["DD_API_KEY"] + + otel_vars = _callback_variables(body, "otel") + assert otel_vars["OTEL_HEADERS"] == _CALLBACK_ENV_FIXTURE["OTEL_HEADERS"] + + +def test_get_config_callbacks_redacts_slack_webhook_urls_for_view_only_admin( + client, auth_as, mock_prisma, monkeypatch +): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_callbacks_config(monkeypatch, mock_prisma) + + webhooks = { + "spend_reports": "https://hooks.slack.com/services/T000/B000/SPEND-WEBHOOK-SECRET", + "budget_alerts": "https://hooks.slack.com/services/T000/B111/BUDGET-WEBHOOK-SECRET", + } + monkeypatch.setattr( + ps.proxy_logging_obj.slack_alerting_instance, + "alert_to_webhook_url", + webhooks, + raising=False, + ) + + def _slack_block(body): + return next(a for a in body["alerts"] if a["name"] == "slack") + + with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): + view_resp = client.get("/get/config/callbacks") + assert view_resp.status_code == 200 + for url in webhooks.values(): + assert url not in view_resp.text + assert _CALLBACK_ENV_FIXTURE["SLACK_WEBHOOK_URL"] not in view_resp.text + view_slack = _slack_block(view_resp.json()) + assert view_slack["alerts_to_webhook"] == { + "spend_reports": "REDACTED", + "budget_alerts": "REDACTED", + } + assert view_slack["variables"]["SLACK_WEBHOOK_URL"] == "REDACTED" + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + admin_resp = client.get("/get/config/callbacks") + assert admin_resp.status_code == 200 + admin_slack = _slack_block(admin_resp.json()) + assert admin_slack["alerts_to_webhook"] == webhooks + assert admin_slack["variables"]["SLACK_WEBHOOK_URL"] != "REDACTED" + + +def test_redact_callback_env_vars_helper_handles_none_and_non_secret_keys(): + from litellm.proxy import proxy_server as ps + + out = ps._redact_callback_env_vars( + { + "LANGFUSE_SECRET_KEY": "sk-leak", + "LANGFUSE_HOST": "https://cloud.langfuse.com", + "DD_API_KEY": None, + "GALILEO_USERNAME": "galileo-user-1234", + "GENERIC_LOGGER_HEADERS": "Authorization=Bearer x", + "GCS_PATH_SERVICE_ACCOUNT": "/etc/secrets/gcs.json", + "SLACK_WEBHOOK_URL": "https://hooks.slack.com/services/T/B/token", + "SMTP_USERNAME": "smtp-user-1234", + } + ) + assert out == { + "LANGFUSE_SECRET_KEY": "REDACTED", + "LANGFUSE_HOST": "https://cloud.langfuse.com", + "DD_API_KEY": None, + "GALILEO_USERNAME": "REDACTED", + "GENERIC_LOGGER_HEADERS": "REDACTED", + "GCS_PATH_SERVICE_ACCOUNT": "REDACTED", + "SLACK_WEBHOOK_URL": "REDACTED", + "SMTP_USERNAME": "REDACTED", + } + + +def test_get_config_callbacks_redacts_email_alerting_vars_for_view_only_admin( + client, auth_as, mock_prisma, monkeypatch +): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps, "llm_router", None) + + fake_proxy_config = MagicMock() + fake_proxy_config.get_config = AsyncMock( + return_value={ + "litellm_settings": {"success_callback": []}, + "general_settings": {"alerting": ["email"]}, + "environment_variables": { + "SMTP_HOST": "smtp.resend.com", + "SMTP_PORT": "587", + "SMTP_USERNAME": "smtp-user-fixture-1234", + "SMTP_PASSWORD": "smtp-password-fixture-1234", + "SMTP_SENDER_EMAIL": "alerts@example.com", + "TEST_EMAIL_ADDRESS": "admin@example.com", + "EMAIL_LOGO_URL": "https://example.com/logo.png", + "EMAIL_SUPPORT_CONTACT": "support@example.com", + }, + } + ) + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + def _email_block(body): + return next(a for a in body["alerts"] if a["name"] == "email") + + with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): + view_resp = client.get("/get/config/callbacks") + assert view_resp.status_code == 200 + for secret in ("smtp-user-fixture-1234", "smtp-password-fixture-1234"): + assert secret not in view_resp.text + view_email = _email_block(view_resp.json())["variables"] + assert view_email["SMTP_PASSWORD"] == "REDACTED" + assert view_email["SMTP_USERNAME"] == "REDACTED" + assert view_email["SMTP_HOST"] == "smtp.resend.com" + assert view_email["SMTP_PORT"] == "587" + assert view_email["SMTP_SENDER_EMAIL"] == "alerts@example.com" + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + admin_resp = client.get("/get/config/callbacks") + assert admin_resp.status_code == 200 + admin_email = _email_block(admin_resp.json())["variables"] + assert admin_email["SMTP_USERNAME"] == "smtp-user-fixture-1234" + assert admin_email["SMTP_PASSWORD"] != "REDACTED" + assert admin_email["SMTP_HOST"] == "smtp.resend.com" + + # --------------------------------------------------------------------------- # GET /config/yaml # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 68122bbba3b..533c37e690e 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -896,7 +896,9 @@ def test_get_config_custom_callback_api_env_vars(monkeypatch): # Bypass auth dependency original_overrides = app.dependency_overrides.copy() - app.dependency_overrides[user_api_key_auth] = lambda: MagicMock() + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234" + ) client = TestClient(app) try: @@ -950,7 +952,9 @@ def test_get_config_returns_email_settings(monkeypatch): monkeypatch.setattr(proxy_config, "get_config", AsyncMock(return_value=config_data)) original_overrides = app.dependency_overrides.copy() - app.dependency_overrides[user_api_key_auth] = lambda: MagicMock() + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234" + ) client = TestClient(app) try: @@ -1007,7 +1011,9 @@ def test_get_config_returns_slack_webhook(monkeypatch): monkeypatch.setattr(proxy_config, "get_config", AsyncMock(return_value=config_data)) original_overrides = app.dependency_overrides.copy() - app.dependency_overrides[user_api_key_auth] = lambda: MagicMock() + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234" + ) client = TestClient(app) try: @@ -1061,7 +1067,9 @@ def test_get_config_cleared_slack_webhook_not_overridden_by_os_env(monkeypatch): monkeypatch.setattr(proxy_config, "get_config", AsyncMock(return_value=config_data)) original_overrides = app.dependency_overrides.copy() - app.dependency_overrides[user_api_key_auth] = lambda: MagicMock() + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234" + ) client = TestClient(app) try: @@ -5205,7 +5213,9 @@ def test_get_config_normalizes_string_callbacks(monkeypatch): monkeypatch.setattr(proxy_config, "get_config", AsyncMock(return_value=config_data)) original_overrides = app.dependency_overrides.copy() - app.dependency_overrides[user_api_key_auth] = lambda: MagicMock() + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234" + ) client = TestClient(app) try: diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index 450391fd503..3fc6bc71b84 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -273,7 +273,13 @@ async def test_config_update_persists_and_reads_back_retry_policy(monkeypatch): assert isinstance(router.retry_policy, RetryPolicy) assert router.retry_policy.RateLimitErrorRetries == 7 - read_back = (await proxy_server.get_config())["router_settings"]["retry_policy"] + read_back = ( + await proxy_server.get_config( + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234" + ) + ) + )["router_settings"]["retry_policy"] assert read_back.BadRequestErrorRetries == 5 assert read_back.TimeoutErrorRetries == 3 assert read_back.RateLimitErrorRetries == 7 From bd9db3691ed7f6d37f4c7cb85243113c20bc2821 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:44:57 -0700 Subject: [PATCH 03/18] chore(lint): remove dead E501 config, fix stale blame-ignore SHAs, note 120 line width (#31927) * chore(lint): remove dead E501 config, fix stale blame-ignore SHAs, note 120 width in CLAUDE.md E501 sat in both lint.ignore and lint.extend-select in ruff.toml; ignore wins, so no line length was linted at all (verified: a 130-char line passes ruff check while T201 fires). Remove it from both lists so the config tells the truth: the formatter's wrap width is the only line-length control, matching how the repo has actually behaved since E501 was ignored in Oct 2024 .git-blame-ignore-revs listed the pre-squash PR-head SHAs for the two ruff reformat commits (#31317, #31518), which never landed on the branch, so git blame ignored nothing. Replace them with the squash-merge SHAs that are actually in history Also document in CLAUDE.md that the line length is 120 (ruff.toml), not 88, so agents stop wrapping to the old Black width * fix: make CLAUDE.md more concise * fix: make the guideline more clear --- .git-blame-ignore-revs | 4 ++-- CLAUDE.md | 2 ++ Makefile | 2 +- ruff.toml | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 7e705ec4f8f..2527239b904 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -13,7 +13,7 @@ 7edf3a9cb55548b143df1692f4ed7c4681d7fcf7 # style: reformat litellm/ with ruff format (#31317) -430b5b8f1b12dc261a49fda99ac5d1b22381a428 +17bfd415aeb5a57fb646b5cc67da1c730aa7c50b # style: unify ruff format width on 120 (#31518) -3dfbeabe626d203ac9de86024519d9a96c484ce4 +48b5a5a0cc5a694a11219416ee0b6eb6e620e74e diff --git a/CLAUDE.md b/CLAUDE.md index a492aabd02d..7d9a6367f18 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,6 +33,8 @@ If you ever make public-facing PR descriptions, comments, issues, commit message Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs +Python max line length is 120, not 88 + Run tests before you commit. Also, run `make pre-commit` right before each commit, which generates types (as needed) and formats/lints your code. Any errors found must be fixed. It only runs when there are staged frontend and/or backend changes and calculates violations, generates types, etc. based on the worktree, so stage what you need or stash/delete unwanted files in litellm/ or ui/ (where backend and frontend lint run, respectively) before running it. If it fails because dashboard api types are stale, it already regenerated them for you. You just need to stage the schema.d.ts, re-run `make pre-commit` to confirm it passes, and commit When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered limits so the ceilings ratchet down instead of leaving stale headroom. It measures the working tree, so it must contain exactly the fixes you're committing diff --git a/Makefile b/Makefile index c3fa21c156c..2cc4ec3e45a 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,7 @@ install-hooks: # Formatting # Wrap width is ruff.toml's single source of truth (line-length = 120), shared by the -# formatter, E501, and the import sorter so there's no 88-vs-120 split to reconcile. +# formatter and the import sorter so there's no 88-vs-120 split to reconcile. format: install-dev cd litellm && $(UV_RUN) ruff format --exclude '/enterprise/' . && cd .. diff --git a/ruff.toml b/ruff.toml index a09bc663ff1..2ea9d7260fb 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,5 +1,5 @@ -lint.ignore = ["F405", "E402", "E501", "F403"] -lint.extend-select = ["E501", "T20", "PGH004", "RUF008", "RUF009", "RUF100"] +lint.ignore = ["F405", "E402", "F403"] +lint.extend-select = ["T20", "PGH004", "RUF008", "RUF009", "RUF100"] # RUF100 (unused-noqa) only knows the rules enabled in THIS config, so it would strip # `# noqa` directives that protect rules enforced elsewhere. List those codes as external # so RUF100 leaves their directives alone: the strict gate (ruff-strict.toml) and upstream From 85f924148a299903fadf568e40b53f8b888016ae Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 1 Jul 2026 19:08:17 -0700 Subject: [PATCH 04/18] fix(bedrock/converse): drop toolSpec.strict for Opus 4.7/4.8 (#31582) (#31923) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(bedrock/converse): drop toolSpec.strict for Opus 4.7/4.8 Bedrock Converse routes Claude Opus 4.7/4.8 through an Anthropic-compatible validator that maps toolSpec to the native tool shape and rejects the extra `strict` key with `tools.N.custom.strict: Extra inputs are not permitted`, even though Anthropic's native API accepts `strict` as a top-level tool field for the same models. Sonnet 4.5/4.6 and Opus <=4.6 accept `toolSpec.strict` unchanged. The existing gate `get_bedrock_base_model(model).startswith("anthropic")` (introduced in #29814 to forward `strict` for Claude on Bedrock Converse) is too broad and regressed Opus 4.7/4.8 callers — see #31582. Replace the inline check with a small `bedrock_converse_supports_strict_tools` helper that excludes the Opus 4.7/4.8 family from strict forwarding. All other Anthropic models on Bedrock keep the existing behavior. Closes #31582. * fix(bedrock/converse): move strict-tools regression to a clean test file The original regression test was added to test_litellm_core_utils_prompt_templates_factory.py, which has pre-existing ruff-format violations throughout (multi-line asserts that fit on one line). The lint workflow runs `ruff format --check` on changed files only, so touching that file surfaces those pre-existing violations and fails CI for unrelated reasons. Move the #31582 regression coverage into a new dedicated test file so the format check stays green. Also collapses the helper's `not any(...)` onto a single line to satisfy ruff format. Covers: #31582 * refactor(bedrock/converse): drive strict-tools gate from model cost map Replace the hardcoded Opus 4.7/4.8 pattern list with a bedrock_converse_supports_strict_tools flag on the affected entries in model_prices_and_context_window.json, resolved via get_model_info with a local cost map fallback, so future models with the same restriction only need a JSON update * chore: revert unrelated credential_migration.py reformat --------- Co-authored-by: ly-wang19 --- .../prompt_templates/factory.py | 9 +- litellm/llms/bedrock/common_utils.py | 47 ++++++++ ...odel_prices_and_context_window_backup.json | 11 ++ litellm/types/utils.py | 1 + litellm/utils.py | 1 + model_prices_and_context_window.json | 11 ++ ...edrock_converse_strict_tools_opus_47_48.py | 107 ++++++++++++++++++ tests/test_litellm/test_utils.py | 1 + 8 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index c2448430387..c1635158d3b 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -5035,15 +5035,18 @@ def _bedrock_tools_pt(tools: List, model: Optional[str] = None) -> List[BedrockT ] """ from litellm.llms.bedrock.common_utils import ( - get_bedrock_base_model, + bedrock_converse_supports_strict_tools, normalize_json_schema_custom_types_to_object, ) from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_defs _valid_json_schema_root_types = frozenset(("array", "boolean", "integer", "null", "number", "object", "string")) # Only Claude on Bedrock honours strict tool schemas; other families - # (Nova, Llama, GPT-OSS) reject the strict field outright. - supports_strict_tools = bool(model and get_bedrock_base_model(model).startswith("anthropic")) + # (Nova, Llama, GPT-OSS) reject the strict field outright. Opus 4.7/4.8 + # also reject `strict` on Bedrock Converse (see #31582) — their validator + # maps toolSpec to the native Anthropic tool shape, which has no strict + # field, even though Anthropic's native API accepts it as a top-level key. + supports_strict_tools = bool(model and bedrock_converse_supports_strict_tools(model)) tool_block_list: List[BedrockToolBlock] = [] for tool_idx, tool in enumerate(tools): # Check if tool is already a BedrockToolBlock (e.g., systemTool for Nova grounding) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 467e1050c99..df432a4d7e3 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -4,9 +4,11 @@ from __future__ import annotations Common utilities used across bedrock chat/embedding/image generation """ +import contextlib import functools import json import os +import re from typing import ( TYPE_CHECKING, Any, @@ -718,6 +720,51 @@ def is_claude_4_5_on_bedrock(model: str) -> bool: return any(pattern in model_lower for pattern in claude_4_5_patterns) +_BEDROCK_MODEL_VERSION_SUFFIX_RE = re.compile(r"-v\d+(?::\d+)?$") + + +def bedrock_converse_supports_strict_tools(model: str) -> bool: + """ + Whether ``toolSpec.strict`` can be forwarded to Bedrock Converse for ``model``. + + Non-Anthropic Bedrock families (Nova, Llama, GPT-OSS) reject the field + outright. Anthropic models forward it unless their entry in + ``model_prices_and_context_window.json`` sets + ``bedrock_converse_supports_strict_tools: false`` — Bedrock routes those + (Opus 4.7/4.8, see #31582) through a stricter validator that rejects the + ``strict`` key on ``toolSpec`` even though Anthropic's native API accepts + it as a top-level tool field. + """ + base = get_bedrock_base_model(model) + if not base.startswith("anthropic"): + return False + flag = _get_bedrock_converse_strict_tools_flag(base) + return flag if flag is not None else True + + +def _get_bedrock_converse_strict_tools_flag(base_model: str) -> Optional[bool]: + candidates = dict.fromkeys((base_model, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base_model))) + for candidate in candidates: + with contextlib.suppress(Exception): + model_info = get_cached_model_info()( + model=candidate, + custom_llm_provider="bedrock", + ) + + flag = model_info.get("bedrock_converse_supports_strict_tools") + if isinstance(flag, bool): + return flag + + model_cost_key = model_info.get("key") + if isinstance(model_cost_key, str): + local_flag = ( + _get_local_model_cost_map().get(model_cost_key, {}).get("bedrock_converse_supports_strict_tools") + ) + if isinstance(local_flag, bool): + return local_flag + return None + + def normalize_bedrock_opus_output_config_effort(model: str, output_config: Any) -> None: """ Normalize Anthropic ``output_config.effort`` values for Bedrock Opus ids. diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b249b951b8e..1a45731e915 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1154,6 +1154,7 @@ "bedrock_output_config_effort_ceiling": "max" }, "anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1203,6 +1204,7 @@ "supports_output_config": true }, "global.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1237,6 +1239,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1271,6 +1274,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1305,6 +1309,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1471,6 +1476,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1505,6 +1511,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "global.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1539,6 +1546,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1573,6 +1581,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1607,6 +1616,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1641,6 +1651,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "jp.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 0e2a6b24806..997498803ac 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -152,6 +152,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_output_config: Optional[bool] supports_image_size: Optional[bool] bedrock_output_config_effort_ceiling: Optional[Literal["low", "medium", "high", "max", "xhigh"]] + bedrock_converse_supports_strict_tools: Optional[bool] class SearchContextCostPerQuery(TypedDict, total=False): diff --git a/litellm/utils.py b/litellm/utils.py index 226b94913b5..b5f77d75b93 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5465,6 +5465,7 @@ def _get_model_info_helper( supports_xhigh_reasoning_effort=_model_info.get("supports_xhigh_reasoning_effort", None), supports_max_reasoning_effort=_model_info.get("supports_max_reasoning_effort", None), bedrock_output_config_effort_ceiling=_model_info.get("bedrock_output_config_effort_ceiling", None), + bedrock_converse_supports_strict_tools=_model_info.get("bedrock_converse_supports_strict_tools", None), supports_computer_use=_model_info.get("supports_computer_use", None), search_context_cost_per_query=_model_info.get("search_context_cost_per_query", None), web_search_billing_unit=_model_info.get("web_search_billing_unit", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9e9523193c7..006c04f5ecb 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1154,6 +1154,7 @@ "bedrock_output_config_effort_ceiling": "max" }, "anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1203,6 +1204,7 @@ "supports_output_config": true }, "global.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1237,6 +1239,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1271,6 +1274,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1305,6 +1309,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1471,6 +1476,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1505,6 +1511,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "global.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1539,6 +1546,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1573,6 +1581,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1607,6 +1616,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1641,6 +1651,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "jp.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py new file mode 100644 index 00000000000..26096c49468 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py @@ -0,0 +1,107 @@ +"""Regression tests for Bedrock Converse ``toolSpec.strict`` forwarding. + +Bedrock Converse routes Claude Opus 4.7/4.8 through an Anthropic-compatible +validator that rejects ``toolSpec.strict`` even though Anthropic's native API +accepts ``strict`` as a top-level tool field for the same models. See +BerriAI/litellm#31582. +""" + +import pytest + +from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_tools_pt +from litellm.llms.bedrock.common_utils import bedrock_converse_supports_strict_tools + + +_STRICT_TOOL = [ + { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "description": "Get the weather for a city", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "unit": {"type": "string", "enum": ["celsius"]}, + }, + "required": ["city", "unit"], + "additionalProperties": False, + }, + }, + } +] + + +@pytest.mark.parametrize( + "model_id", + [ + "bedrock/us.anthropic.claude-opus-4-7", + "bedrock/us.anthropic.claude-opus-4-8", + "anthropic.claude-opus-4-7", + "anthropic.claude-opus-4-8", + "anthropic.claude-opus-4-7-v1:0", + "bedrock/eu.anthropic.claude-opus-4-8-v1:0", + "bedrock/global.anthropic.claude-opus-4-7", + ], +) +def test_bedrock_tools_pt_strict_dropped_for_opus_47_48(model_id: str) -> None: + """Opus 4.7/4.8 on Bedrock Converse reject toolSpec.strict — must be dropped.""" + result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id) + assert "strict" not in result[0]["toolSpec"], f"strict leaked into toolSpec for {model_id}: {result[0]['toolSpec']}" + + +@pytest.mark.parametrize( + "model_id", + [ + "anthropic.claude-sonnet-4-5-20250929-v1:0", + "bedrock/us.anthropic.claude-sonnet-4-6", + "bedrock/us.anthropic.claude-opus-4-6", + "bedrock/us.anthropic.claude-opus-4-5", + ], +) +def test_bedrock_tools_pt_strict_kept_for_other_anthropic(model_id: str) -> None: + """Sonnet 4.5/4.6 and Opus <=4.6 accept toolSpec.strict — keep forwarding it.""" + result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id) + assert result[0]["toolSpec"]["strict"] is True, f"strict missing for {model_id}: {result[0]['toolSpec']}" + + +@pytest.mark.parametrize( + "model_id", + [ + "us.amazon.nova-micro-v1:0", + "meta.llama3-2-11b-instruct-v1:0", + ], +) +def test_bedrock_tools_pt_strict_dropped_for_non_anthropic(model_id: str) -> None: + """Non-Anthropic Bedrock families reject toolSpec.strict — must be dropped.""" + result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id) + assert "strict" not in result[0]["toolSpec"] + + +def test_bedrock_converse_supports_strict_tools_helper() -> None: + """Direct check for the gate helper used by factory.py.""" + assert bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-opus-4-7") is False + assert bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-opus-4-8") is False + assert bedrock_converse_supports_strict_tools("anthropic.claude-sonnet-4-5-20250929-v1:0") is True + assert bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-opus-4-6") is True + assert bedrock_converse_supports_strict_tools("us.amazon.nova-micro-v1:0") is False + assert bedrock_converse_supports_strict_tools("") is False + + +@pytest.mark.parametrize( + "cost_map_key", + [ + "anthropic.claude-opus-4-7", + "us.anthropic.claude-opus-4-7", + "anthropic.claude-opus-4-8", + "us.anthropic.claude-opus-4-8", + ], +) +def test_strict_tools_flag_set_in_model_cost_map(cost_map_key: str) -> None: + """The gate is driven by ``bedrock_converse_supports_strict_tools: false`` in + ``model_prices_and_context_window.json``, not hardcoded model patterns.""" + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + cost_map = GetModelCostMap.load_local_model_cost_map() + assert cost_map[cost_map_key]["bedrock_converse_supports_strict_tools"] is False diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 28b82cf035b..2432b377d54 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -858,6 +858,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "type": "string", "enum": ["low", "medium", "high", "max", "xhigh"], }, + "bedrock_converse_supports_strict_tools": {"type": "boolean"}, "tpm": {"type": "number"}, "provider_specific_entry": {"type": "object"}, "supported_endpoints": { From c4f28ce2874f82473f171d055335c322ba5cf31b Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 1 Jul 2026 19:11:18 -0700 Subject: [PATCH 05/18] fix(bedrock): trigger Nova Sonic generation on response.create so realtime sessions stop hanging (#31924) * fix(bedrock): trigger Nova Sonic generation on response.create so realtime sessions stop hanging (LIT-2239) * fix(bedrock): reopen audio content at client sample rate after trigger block * test(bedrock): cover realtime handler disconnect flush and stream-end guard * fix(bedrock): always close realtime input stream even if close flush fails * fix(lint): use contextlib.suppress in bedrock realtime cleanup to satisfy BLE001 budget * fix(bedrock): suppress bedrock close send errors per-message so promptEnd/sessionEnd still flush --------- Co-authored-by: Cursor Agent --- litellm/llms/bedrock/realtime/handler.py | 37 ++- .../llms/bedrock/realtime/transformation.py | 146 ++++++++- .../llms/bedrock/realtime/trigger_audio.py | 208 ++++++++++++ ruff-strict-budget.json | 4 +- .../realtime/test_bedrock_realtime_handler.py | 174 +++++++++++ .../test_bedrock_realtime_transformation.py | 295 +++++++++++++----- 6 files changed, 768 insertions(+), 96 deletions(-) create mode 100644 litellm/llms/bedrock/realtime/trigger_audio.py create mode 100644 tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index 6db2571090a..557ee3348d5 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -5,6 +5,7 @@ This uses aws_sdk_bedrock_runtime for bidirectional streaming with Nova Sonic. """ import asyncio +import contextlib import json from typing import Any, Optional @@ -156,12 +157,19 @@ class BedrockRealtime(BaseAWSLLM): session_state: dict, ): """Forward messages from client WebSocket to Bedrock stream.""" - try: - from aws_sdk_bedrock_runtime.models import ( - BidirectionalInputPayloadPart, - InvokeModelWithBidirectionalStreamInputChunk, - ) + from aws_sdk_bedrock_runtime.models import ( + BidirectionalInputPayloadPart, + InvokeModelWithBidirectionalStreamInputChunk, + ) + async def send_to_bedrock(bedrock_message: str) -> None: + event = InvokeModelWithBidirectionalStreamInputChunk( + value=BidirectionalInputPayloadPart(bytes_=bedrock_message.encode("utf-8")) + ) + await bedrock_stream.input_stream.send(event) + verbose_proxy_logger.debug(f"Bedrock Realtime: Sent to Bedrock: {bedrock_message[:200]}") + + try: while True: # Receive message from client message = await client_ws.receive_text() @@ -176,19 +184,15 @@ class BedrockRealtime(BaseAWSLLM): # Send transformed messages to Bedrock for bedrock_message in transformed_messages: - event = InvokeModelWithBidirectionalStreamInputChunk( - value=BidirectionalInputPayloadPart(bytes_=bedrock_message.encode("utf-8")) - ) - await bedrock_stream.input_stream.send(event) - verbose_proxy_logger.debug(f"Bedrock Realtime: Sent to Bedrock: {bedrock_message[:200]}") + await send_to_bedrock(bedrock_message) except Exception as e: verbose_proxy_logger.debug(f"Client to Bedrock forwarding ended: {e}", exc_info=True) - # Close the Bedrock stream input - try: + for close_message in transformation_config.session_close_messages(): + with contextlib.suppress(Exception): + await send_to_bedrock(close_message) + with contextlib.suppress(Exception): await bedrock_stream.input_stream.close() - except Exception: - pass async def _forward_bedrock_to_client( self, @@ -206,6 +210,10 @@ class BedrockRealtime(BaseAWSLLM): output = await bedrock_stream.await_output() result = await output[1].receive() + if result is None: + verbose_proxy_logger.debug("Bedrock Realtime: Bedrock stream ended") + break + if result.value and result.value.bytes_: bedrock_response = result.value.bytes_.decode("utf-8") verbose_proxy_logger.debug(f"Bedrock Realtime: Received from Bedrock: {bedrock_response[:200]}") @@ -252,6 +260,7 @@ class BedrockRealtime(BaseAWSLLM): except Exception as e: verbose_proxy_logger.debug(f"Bedrock to client forwarding ended: {e}", exc_info=True) + finally: # Close the client WebSocket try: await client_ws.close() diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index 498567a4ecf..fe5f0584e03 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -4,14 +4,18 @@ This file contains the transformation logic for Bedrock Nova Sonic realtime API. Transforms between OpenAI Realtime API format and Bedrock Nova Sonic format. """ +import base64 import json import uuid as uuid_lib from typing import Any, List, Optional, Union +from pydantic import BaseModel + from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig +from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm from litellm.types.llms.openai import ( OpenAIRealtimeContentPartDone, OpenAIRealtimeDoneEvent, @@ -35,6 +39,17 @@ from litellm.types.realtime import ( from litellm.utils import get_empty_usage +class BedrockContentEnd(BaseModel): + stopReason: Optional[str] = None + + +TRIGGER_AUDIO_SAMPLE_RATE_HERTZ = 16000 +TRIGGER_AUDIO_BYTES_PER_SECOND = TRIGGER_AUDIO_SAMPLE_RATE_HERTZ * 2 +TRIGGER_LEADING_SILENCE = bytes(TRIGGER_AUDIO_BYTES_PER_SECOND // 2) +TRIGGER_TRAILING_SILENCE = bytes(TRIGGER_AUDIO_BYTES_PER_SECOND * 3) +TRIGGER_AUDIO_CHUNK_SIZE = 1024 + + class BedrockRealtimeConfig(BaseRealtimeConfig): """Configuration for Bedrock Nova Sonic realtime transformations.""" @@ -43,6 +58,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): self.prompt_name = str(uuid_lib.uuid4()) self.content_name = str(uuid_lib.uuid4()) self.audio_content_name = str(uuid_lib.uuid4()) + self.prompt_started = False + self.client_audio_streamed = False # Default configuration values # Inference configuration @@ -247,6 +264,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): prompt_start = {"event": {"promptStart": prompt_start_config}} messages.append(json.dumps(prompt_start)) + self.prompt_started = True # Send system prompt if provided instructions = session_config.get("instructions") @@ -304,8 +322,22 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): List of Bedrock format messages (JSON strings) """ verbose_logger.debug("Handling input_audio_buffer.append") + self.client_audio_streamed = True messages: List[str] = [] + if hasattr(self, "_audio_content_started") and self._audio_content_sample_rate != self.input_sample_rate_hertz: + mismatched_content_end = { + "event": { + "contentEnd": { + "promptName": self.prompt_name, + "contentName": self.audio_content_name, + } + } + } + messages.append(json.dumps(mismatched_content_end)) + delattr(self, "_audio_content_started") + self.audio_content_name = str(uuid_lib.uuid4()) + # Check if we need to start audio content if not hasattr(self, "_audio_content_started"): audio_content_start = { @@ -329,6 +361,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): } messages.append(json.dumps(audio_content_start)) self._audio_content_started = True + self._audio_content_sample_rate = self.input_sample_rate_hertz # Send audio chunk audio_data = json_message.get("audio", "") @@ -383,7 +416,6 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): List of Bedrock format messages (JSON strings) """ verbose_logger.debug("Handling conversation.item.create") - messages: List[str] = [] item = json_message.get("item", {}) item_type = item.get("type") @@ -392,6 +424,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): if item_type == "function_call_output": return self.transform_conversation_item_create_tool_result_event(json_message) + messages: list[str] = [] + # Handle regular message if item_type == "message": content = item.get("content", []) @@ -443,6 +477,12 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): """ Transform response.create event to Bedrock format. + Nova Sonic only starts generating after it detects user speech, so text-only + sessions never get a response on their own. Injecting a short spoken "ready" + utterance (followed by silence) makes the model respond to the pending + interactive text input. Sessions where the client streams its own audio rely + on Nova Sonic's built-in turn detection instead. + Args: json_message: OpenAI response.create message @@ -450,8 +490,53 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): List of Bedrock format messages (JSON strings) """ verbose_logger.debug("Handling response.create") - # Bedrock starts generating automatically, no explicit trigger needed - return [] + if not self.prompt_started or self.client_audio_streamed: + return [] + + messages: list[str] = [] + if not hasattr(self, "_audio_content_started"): + trigger_content_start = { + "event": { + "contentStart": { + "promptName": self.prompt_name, + "contentName": self.audio_content_name, + "type": "AUDIO", + "interactive": True, + "role": "USER", + "audioInputConfiguration": { + "mediaType": self.input_media_type, + "sampleRateHertz": TRIGGER_AUDIO_SAMPLE_RATE_HERTZ, + "sampleSizeBits": self.input_sample_size_bits, + "channelCount": self.input_channel_count, + "audioType": self.input_audio_type, + "encoding": self.input_encoding, + }, + } + } + } + messages.append(json.dumps(trigger_content_start)) + self._audio_content_started = True + self._audio_content_sample_rate = TRIGGER_AUDIO_SAMPLE_RATE_HERTZ + + messages.extend(self._response_trigger_audio_messages()) + return messages + + def _response_trigger_audio_messages(self) -> list[str]: + pcm = TRIGGER_LEADING_SILENCE + ready_trigger_pcm() + TRIGGER_TRAILING_SILENCE + return [ + json.dumps( + { + "event": { + "audioInput": { + "promptName": self.prompt_name, + "contentName": self.audio_content_name, + "content": base64.b64encode(pcm[offset : offset + TRIGGER_AUDIO_CHUNK_SIZE]).decode(), + } + } + } + ) + for offset in range(0, len(pcm), TRIGGER_AUDIO_CHUNK_SIZE) + ] def transform_response_cancel_event(self, json_message: dict) -> List[str]: """ @@ -467,6 +552,35 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): # Send interrupt signal if needed return [] + def session_close_messages(self) -> list[str]: + """ + Build the Bedrock events that gracefully close the session + (contentEnd for any open audio content, promptEnd, sessionEnd). + + Returns: + List of Bedrock format messages (JSON strings) + """ + if not self.prompt_started: + return [] + + messages: list[str] = [] + if hasattr(self, "_audio_content_started"): + audio_content_end = { + "event": { + "contentEnd": { + "promptName": self.prompt_name, + "contentName": self.audio_content_name, + } + } + } + messages.append(json.dumps(audio_content_end)) + delattr(self, "_audio_content_started") + + messages.append(json.dumps({"event": {"promptEnd": {"promptName": self.prompt_name}}})) + messages.append(json.dumps({"event": {"sessionEnd": {}}})) + self.prompt_started = False + return messages + def transform_realtime_request( self, message: str, @@ -837,10 +951,11 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): Optional[ALL_DELTA_TYPES], ]: """ - Transform Bedrock promptEnd event to OpenAI response.done. + Transform a Bedrock end-of-response event (promptEnd, completionEnd, or an + END_TURN contentEnd) to OpenAI response.done. Args: - event: Bedrock promptEnd event + event: Bedrock event that ends the response current_response_id: Current response ID current_conversation_id: Current conversation ID @@ -848,7 +963,18 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): Tuple of (events, reset_output_item_id, reset_response_id, reset_delta_type) """ verbose_logger.debug("Handling promptEnd") + return self._response_done_events(current_response_id, current_conversation_id) + def _response_done_events( + self, + current_response_id: Optional[str], + current_conversation_id: Optional[str], + ) -> tuple[ + List[OpenAIRealtimeEvents], + Optional[str], + Optional[str], + Optional[ALL_DELTA_TYPES], + ]: if not current_response_id or not current_conversation_id: return [], None, None, None @@ -1084,6 +1210,14 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): current_delta_chunks, ) returned_messages.extend(events) + if BedrockContentEnd.model_validate(event["contentEnd"]).stopReason == "END_TURN": + ( + done_events, + current_output_item_id, + current_response_id, + current_delta_type, + ) = self._response_done_events(current_response_id, current_conversation_id) + returned_messages.extend(done_events) elif "toolUse" in event: events, tool_call_id, tool_name = self.transform_tool_use_event( @@ -1093,7 +1227,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): # Store tool call info for potential use verbose_logger.debug(f"Tool use event: {tool_name} (ID: {tool_call_id})") - elif "promptEnd" in event: + elif "promptEnd" in event or "completionEnd" in event: ( events, current_output_item_id, diff --git a/litellm/llms/bedrock/realtime/trigger_audio.py b/litellm/llms/bedrock/realtime/trigger_audio.py new file mode 100644 index 00000000000..5783dae54bb --- /dev/null +++ b/litellm/llms/bedrock/realtime/trigger_audio.py @@ -0,0 +1,208 @@ +""" +Pre-rendered spoken "ready" trigger audio (16kHz, 16-bit, mono PCM), generated with Amazon Polly. + +Amazon Nova Sonic v1 only starts generating after it hears the user speak, so text-only realtime +sessions inject this short utterance to trigger a response (same approach as Pipecat's +AWSNovaSonicLLMService assistant-response trigger). +""" + +import base64 +import gzip +from functools import lru_cache + +READY_TRIGGER_PCM_16KHZ_MONO_GZIP_B64 = ( + "H4sIANGpRWoC/517dXQcR9Bnw+Duis3MzMwkc8xsxxTZMTMzM0PMmJhBjpmZmWKQZSaxFrQ80H0lJXf3vXf/nev17ExPQ3Xh" + "r2wPQv8/f/D/uMP/zxv8f64YkSyiWU1AIpCcRSqyICtQCApF4SgSRaHsKCfKjfKhgqgIKo5KobKoPKqEqqFaqC5qgBqjFqg1" + "ao86o+6oF+qDYtAgNBSNQKPQODQeTUZT0Uw0B81D89ECtAgtRcvRMrQCrQRahVajNWgtWo/+QBvQJrQRbQbagraibWg70Da4" + "2/rf7zbozxyxCcaug1krYZXFaCGai2ajGWgKmgi7jYFdh6ABqB/qjXqgLqgd8NUcNQUe66CaqDJwXQaVhDMUgNNkh7NZ4bSc" + "69zLXTyVJ/Kv/BN/z9/w5/w+0B1+g1/kZ/gpfoL/zQ/xA/wvvovv5Jv5Fmjr+Tq+lq/iS/kioPl8Fp+eRRP4KKBhfBDvw3vw" + "Lrw978jb8Fa8JW8B1JQ34PV5Q2iZ1wa8Ea/Da/Ja0Gpm9TTmzXgT3hzGt+a/wNz2vBPM78TbwtyW0NuIV+fVeCWgCrwyUHmg" + "yll9NWClOrwurNA0a42GMLYerFoN3pf7j8rwYkBFeWFehOcHKshz8dw8Ow/nUTyCh/IwaDYuc4mrPATIBs8WeJMNxmTnOWF0" + "Tp4DnsK4lYscc8405mNulsFc0FKYg9lZKjQX3KXDcxJck+E5haWxRPYd2k/2hX2G9gkonr1nX6H3C7RkGOmB1QjsGckLAK+1" + "QAqdeQwfD7Jdz3fzk/we/4f/5D4uoVyoGOizOdjbMLCw+WBBO1Esuo7uoZfoE/qJnCgARq7iUByGs+G8OGcWReAQbMUCDqBk" + "FI9eoGvoGFjVSrCdAWAr9VBhsHsHf8EP87m8F8gV88dsBxvFajKJvTS3mjFmOdNnXDMWG12NUoZPf6jv0afpnfRKejadaena" + "B+219kqL0xK0oBamV9E761P1I3qcLhhVjL7GEuOckWAUMHube02nWZ/NY7dYBFjJZR4JNnsRqbgr3o7f4WykNZlJDpEH5AfR" + "SRgtQMvTarQJbUZb0JZA7Wjr/1o7+gs8N6K1aRmal9qoSRLJC3KF7CHLyUTSl7QhtUkxkptkIyEklOQghUgF0oh0IINg/WVk" + "O9lLjsI+u8kfZC4ZQ/qQZqQ6jM9DchIbkQglfpyKP+Cb+DDejBfiEbgDrogx/gftQWNRbaTx23wBjwYbuMwmsfLsq7nZ7Ghi" + "84IxHs7r1A/rA/Wiery2UeukRWiPg4uD0UEeuBSYFWgQEAPP/dv9k/xd/K387fy9/DP9x/2p/mqB1QEjMCvIg/M1m75Tr2m8" + "NuaZ5dhrNoaHQnzIDpzUIzdJLXqS5hEmCVcFJpQXu4gjxelAk8XBYgexgiiLP4XLwmphkNBAyCkk0yt0Be1HK1JCX5JdZDSp" + "D2d7htfhLmAXX9FR0H5XVAPlQDp4/xN+BezsCHj6Xrge4cfA9y/wS/w6nPUxWN97/h2ihIObXISoWASiXwxEtDPIjRriTTiA" + "B5N3pD19RH8RbgilxWXiZ7GI1EUaLU2QxkojpcHSUGmGtEO6KX2UXkvbpYaSQ7wq/iXuEleKQ8XS4gdhsVBXEISjoNd3ZBzJ" + "S27hCbgMfgeRTUGHeXfuY6tZFfbQ7G+mGmOMJH2UTvRNWnZtfjA8eBjk+srfx+/x7fQ18/m8+739veW8yPvec8nzl2ejZzXQ" + "Ns9pz2dPiLeld4+3iO+pb6t/RWBr8IL2UY8wm7Pp/AC6huPIF/pOuC1ul4bKuZVDSnF1kfpUDbPUs/xq6WmpaZEtD9TJaqi6" + "QymlHJAj5KnSM7GYOF64SYvTtSSC7MB1cALE6WbIC/GyG8QaESJKAfDm2TyWx3EZInAryA9LQfZvwOor4c54PF4B8juGn+A0" + "HEWqgAf0JkPIeDKFTCOryA5yiaQSG42mI+hf9DvNLvQXrgg2sZ+4V/wilpH6SFulT1JReYJ8WU6QsynFlbxASPkiX5S3yNPk" + "0fJkebm8RO4ox0lNpbNia/Gj0BEsYxxYhZ1cJOdJE3IWM8h0vfgpFsVGmaeNaOOL3kK/ohXV2gfnBL74R/q/+rr57N6V3ibe" + "gGevp4fnm3uuu7T7bcaJjJkZTTMsGfddm11jXF1dfVyzXBdcpTOeZFx2f/HU8j33rww207Obz9l0VJqcoBXEQ1KyXEgtYgla" + "TlgL2ZbZ/rGl2GwheUKUkHhbN9t1a6S1iWWM2kepJxeTUoQLdD5pgWuiWnwWe282Ac3vNhZBNBllTDBWGDuNNwYzmpixZgTb" + "ysqBhJtDDOyL43E0nCw/HUPP0jTwlipCKaGQQIR4eotupcNoPcrJY3KP3CEmaUKnUZ22F24KMeItcbKUXf5DrqUsUxxKe3Wc" + "+lBNVQVLW8twy2kLs2S3NrY2tFqsryytLN/VieoLpaEyV/ZLG6VGUlDcKo4VO4uR4mOhs3CI/iQyyY3t/CmLNfcaF/TT2sXg" + "+0BUoLd/r4/4fvde80ieTu6/M1Jc0a5LzvzOBY4P9qH2bPaE9Pj0T+mivaZ9sv20XbfncNR3vHY8dHbPWOQp6v9VSzXX4wpi" + "HWWK5Zw1xtbd9t46yeq0PLHkt26y9rXlDukSEh1yz5ZgPWwppw6VN4qKsIL0wQPQJH6WtQH9Kmwge8em8ZFoNP6NtKDXAf5E" + "Co/AunLQzPiZi26gFYTzgiSGiYeEdHqKNMKneW0WahY2Ruo+7Yi2SzsBkT6P3lZfol/Tdb2+McLYblw2TNDEA/MhU5FANgnD" + "5BWWniEnw6ZHDIqcGZk7ck7EzfAe4ZXDK4WvCufhvSPORFyJGBnxPJyHpYW+DEm1mdYm1o2WoDpCjVOqKmvlEvJtaaqUXWom" + "NhLiSS5cmzczF+rZtBzBsYEE/xV/G/8nXz/feu87j+zp5l6TkTfjlquFq4DrunOUs6bzo2OsQ3Vst3ezlwdJNrBXg2uUXU8v" + "YRccp5293WX85fSd/C4tIt9UW1jnW7mljWWk2k5NV1Msw21KaL6wa2F/hrUPXWOba/lFmShlE2/SPeQNfoZsaDS/xtYyCRAS" + "Q71IN+qhbYRswl+0Ab1N/OQcfSa0l47Km5X3ynilh/xSjBIakivoL/6aDWcCm21+MmoYO/WC+m1tnrZUO6p91urq8/THeg8j" + "YHw28/Ih6DO+TW+KcfJE4KVw6O2wHeHFImIi9PDocCNsadiksPSwjLA3YTfCqoZ1Cz0fcsrmsi6ydrXOtn6z5rAxqwJetNBa" + "w/qPpYflndoRYtVpeYpUW/TRu2QktqKebJtxR7se8Phue85nNHGZjuoOh32MvYy9kP1D+iOgielqesv0kukF0wdCs6ZPT7+V" + "Xsf+1Z5gz+ko6Ryc8dHr19aiuaKpbrC1BaA02rpcfaNUV7tbGtkehFQJ9YSk2aKsK9SH8hWxibCFzEM9WG/Dqbm02UYCy0W+" + "CbOlmfJ4eZf0U7whrpAaKVUsH6wOW/mQt7bBtlfWn5azqqjYxao0G+pu1tK7a39px/U4Y7hZzaxvJGoDg7kCp/yJ/seBGE00" + "jplTeE9cmhYSp0mfpXjJK54RHgmzxDxyjCU8pFrYwvBLYUbIV2tXdYjslC5IyZJPPqUsVwOWYrZFIQmhxcECK4d+tcVa21kr" + "2mJDDobVi4iNiIooGUZCcljnqcWUSPmBGCGEkv68jtle7x787v/V53UnuD44CtuT0r+m77B3dMQ4tzvDnDmcC+3v01ukR6Tt" + "TU1NiUlpnKKkzk4bm97GUdn5wFHa2cQx3jnE0zJoQy7Rp26x9rP+Zsmh7lJKKHOVEeoPJUbpLn8SL4rVwffO0R0oYAwPCv6u" + "vqf+/noV/ogchlKFC7eFH+JB+YCabmliZdZctiW2DiH3Q1qFjg2Js2ZT44VpaKJRVOOBtYGMwG/BaUF74GwgIrg9uF7bodcF" + "9HZTb2M8Z+GkjlhfGaJ6lRVyZbEEnYC9fBZCdKT4WZYtsyDnzVRtymepqFQTOLog51Z3WIbZzoa8CG0YViwsLLRqyGXbUlux" + "0Blh9cIPRoRHvo14H/4+9KXtpkVTHssnpcLSafEvYSe1ksm8tnldEwN7vf3d91yHXYMztrtPex54FnoKu486v9u/p81Lq2Zf" + "51jhDHfddaxOb5w2K+Vz0vDk2sk1UtypQ1MrplZKr5ihBYeRnmopm2RrrtaX5otLxFNSYbWvdZ/NbZmjZJPzilNoE4xMR6CJ" + "d7a7jO+e9pYdJctETTwtxNB89K6QJEdaZ4WsCWsUPjl8emT5qKURR0K3qIwW5tX1ZsHHwdMQ15jWMVjMH+mb7X3oUwLvA9O1" + "3kYF3pRUFB/LLdVcahl5s5CPHAL0nIRvkXByGM1k7dkhXhDF4TkkkgSIRNcLprRV2WM5bb1pfWxZot5TvZarNlfIr6GvQ3rb" + "zlrXWivbqobst32wVrW+VAurMapb3Q8Svw6RppVYXahEz5JpaIy5WGvk/8tb1fsNUEwzb0fvBPdJl+o8YGfpetqxlE7JD5OV" + "ZHeyJd2bylJbptLEbgkzEub8nJrUPqUa2F2T9J0Zn/RE8aytctigkHVyfmrisUKI8kW9Z+mulpSGkXGsvDnUSAi6vH9nrHYd" + "zNjjL8C60fFSXmm+MFxoJc6QhlpWhA4Mt0SWiwyNWBs+MnS+db9yQtjO8xjN9ViwpSFGV+2Cv5D3c0ZOT/NAV8PkB/FG8oHO" + "EHaJQ+W2ynO5gtiQLIQKWoUKeTeqz18YX4I5AgW110YaTyd++kpoK42RXyjvLZutO61rbKlwHWfbZntqq2UbaTmtDJDPSivE" + "Z+Im+Z1yXm2reuWncmu5lrxIvqx8VzR5tXwEUO99uh3FsbV6seBNX3VvVY+asTVjbUaM+45nrPuOa7Kjpb1C6viU96lbU3+H" + "bLAqvaj9YdqQVG9S++SMpEpJ+ZOvJfZPupU8zLFBmyuWCskRut26Rub4LkvlbmGuckC9r1SUDpJu5qLACt8Zz09XlPOKc73n" + "czBonsPRtBO5C7V/FD0rb7A5w/dGXc/2W9THsDO2CEsZOU24Txriv9EP/oYdMO/q6wKPvfc8m7xlghfMmmSdWFdC0kBxn9hf" + "6iwtkBLFPmJNcZjgpnXJYtSLbdE2Bry+rv7DOkVf6BbpqhQQZ0tHpRRptdoSLOpL6PzQyyFNbYssJRVBuihECcWFC5Ku/KME" + "lFFKafmlRKSOUi55plLUkmLxWH9YTXWaPFdcCRmpBA81rgU/Brr48/sEr83TMyObe2LGtYyKGdQlgVfOs/9m/9W+1RnpLOTa" + "6yzuLOOw2KemD087n5or7XJqVNrxlI2OzcEmwjXLGltTWz7rXsmHR6D2Qg85XGmu9FduSHVJvLbYW9Q9OKOFu6N/oLGTj0WD" + "+RjeEkXjz2SfOF5Ntk0NC4scEtUj8n3IA6Wl+E4whfeCIrYQPuKnrKYR0P4MouDvwWjdyk+SseIieaKsgLzaiGWl3+UcgL/t" + "4kmxoxCPu3HNGKiN97cEVHAv8IFtIa3EpVJuqb+0Uv6hlLXeCu0R3i78aHjZ8KuhD6wB5Z3UX2wijhFVmaiXFEWeJM2VqskN" + "lGfqaUtXywBLJ8tjS0OrWx0sj4DYeAQ/Rk9YUb2Xv41HdI/wxPk3BqoHznkPuG46cju+2rO7Jrlfuld5Orh7uMY6UtNcKcVT" + "XqRMs/d2fAFLy5WamPwlMT4h2rmcpakTbLct9eQUWhg9QOWkebarIdstneUV4lK0UkvwXM8o5U7y7AycMP38lrkkuMbPgkvM" + "R/ik/MV6JvRC2OKw26HlQ2pZtsrvpMUKs3yw3Fbmih5eTTvvzevt6m+vjTFGQs1o5fXYXwyjFrgHnSp2lgeotS0n1HxKfekG" + "nYldPC8aiUrxDuYzbV1wj57MMvA3GifIYkF5gtJC/R3QbnnrGdvvIYNCRoSsty2zrJEjxDHCNGGcmENi4haxkVhQbCkmin+K" + "1aQSck8lQemozlOGSSWFFqQCNnkzNkDPHsjh++p54tnofe4p7NnnphkhzucO2RHpPORIcrRycecl++r0hmkd0i6nXUvLbk+2" + "b007n9wv0f+z18/eSdczTqCnwEV1eSjNiYuwYby1MMYSb8WW5/IF8Q46qKV6DrivePr79wSjjFxGpDbKN80bHqhp3ECquFZ5" + "py5XO6uJ6lJLH6jdHljKWT+ABMrL+ehD9lgrGiyiLTT2mR+Nq3p+QJUHg80NiU/A2+lyyCo7lA9ST7G4oJFtpAytJ8wX2pN1" + "bKNeWauqt2Nf8HlaR9SEUmJJKb/cTj6gRFjyWX22waHZQh/aBloqKWmARLLJY+Rlsk1OETeLzaTfpUZyqNxdKg4I5Kh0XEwX" + "2gr5hFPUiVvxFMMZXO2zeX96Xnp+95z0rPBU8sx1j3P5HGccVR0THCcc+ZzvHbvBzkTHqfS/0lhq7rRHqS9SHiUfT56WJCVO" + "TinnsbM4saFcU/ydKLgl7k5LyonKCbWepaKlitKcxurEl99d2X3Jt16/ZlbVW/o2ua+5a/h6Bp+bY8k46aO62/YjZKtttGW0" + "ck8pad0Y2jS8avhj2yC5Iz5ohhvT9C/aJL2u3kZfpGXTbgS1wOBAC20qy0PGCNehIh1J+9PG9D3dJBQUDNwY70EneF+eyErx" + "bmgk/5v7UVN8nRQUZovD5EbKW6WQpT6g7t8tFdWDUGVvV+up1dRR6gGlkdJPOSvHS3OkKOmq6Befi7PEOKjLiJhIV5BsuBeP" + "NQO6qB/S2gUf+Gv5Zd8nTzvPM3fpjDEZbd0lM+yueq5RjvaOk/Yj6eUcOR2b06um9Ujbl/otJZAyJmVHct/k8Ylm6iNfPL8m" + "vBaX0p1ordnabEZqWVjotfAjEQNDk6RkZg9sDEzQn7JCZCfpx5r4H7lyu2q7b/mi9EW8HDWB1gq9hWV0jlBL2WcLhPYM/xg+" + "OmyRdbtUmJbDyxGF+nMiyYcKGPmCBX2LQKtj/cP10eCPV9l9VpdNNZuzs+gQHSzekcbJ/SQH7Yr38VH8Mxol7FBmWpLU29IY" + "KuMfqBn9JtUCpDrBgi1vlbbyWOkPaay8Us1u7R1SPjTNWly9IX2Vsit31G7WrdZfLS3Ur1I5KVaYT0fSKjSO7MLvmUNvHGwW" + "OOO/4nvtb+Xv7D/rO+dxZSxxDnDUdPZ3+jOKeVhGWsZOxxzHLOcNRzVHFfu+tI72nvYSzvN2nLYh+U5i/bRX3oJsjbBOGMP3" + "aFv9/Q2H8FvInPDVYTVsWP1NmihZAOk9JUfJCuGm+EkYgmP1OP8dL/Jv1xawW7gj+Qc/RJfZIfYnukvbyolqYVunkCe2RdZK" + "lpxqbzla2gtoNsz6QE0QFdzD1PQocx27z/2skDlK9wQHBI8Ex2pD9InGZvMMX49aoVb8EPOa7dkPvpjWlG+rpS3V1WxyEbCT" + "dKmv+tLa1bbF2tkSq4yS8ggzSXVynJwRFkh15GxSSZqEuqC9uK2wTFovj5SvSjOlN2JRsZ+whBakaaQSbUkvkTr4BDtrtDRK" + "GgWMMMOr/2oc02toAwMj/O39NPA60Dp4NoACjfzcN8g/33/eX9Pfz7/LPzJQIdDN/8i71PvIq/na+6f6fb7FvrK+VpDB6wXL" + "6T2NXUayXkq36KqxyVzBf6ApuC9ug/fg2aQY7Uo70qm0mNBHHCqFyPelUCm/+EYYLo6Ucsqq3FzaJVYRK4klxThhidBYaCa0" + "FiYLc+HuJq1I15NhZBcJp9VobjqHfMF/4K34LN6HS+JDaACajY4CXv0FHectuMRdzMds/CNY4ho2iDVi9Vg1VpR9M/+B6h2z" + "7mw2W88WsXXsFrPwxfwHb4I+o214FqlOT9FywkqhipgkXpNmyW65PlS4awFreJXcaj91rjpWbawSNZtaDjDtVSW7ckPeJJ+Q" + "H8lz5ZbyFSlacon7xV/FIuJrYa1QWfhEe9PDgJhHYILvAzYcjt7zXHwy1MjXzbNmZ/O2Ud9I0LkeYvyp59evaCF6V/2HVl+7" + "FFS1odo+bZWWR3MGfcHcWh+oD19oH7Uw3dSi9G76Dv2+Lhn9jM9GTXOYudzMxY6z3jyBt0NnUGmcgJeTNyQDNL2KZhOmCjeF" + "/cJzIVVIFz5BZXUCetqAVOsKOj0K8hxKB9ESdCqpSsaTg2Qy0fGvuDy+jQlZgp+hTegaKoPD8E70htfgE3gldB81Rvl5GkM8" + "hl/ij/g4OFNP/op3Qh3Rc76fS+hPFIeeo1j0BMWji8BREfwPfoR34qW4J/6EB0HkOIcL4J7ER6uLTQERHRMKiUfF/NI06ZM0" + "UE6Rdfmn7JQny4XlgDRIVpQIpZscKp0SPdJguZvwinQVeymX5FhiomJ0txAmuPAEEi3cEeNJB3bfIKiqsFFYgzqbETw77sH7" + "m3HGbUZxU15AfxJYbj4jt9DcQE1Pu8AEQA+GVs/XxP3I89pX3dfWc8izxVfQ9ynjJkTWFs4Qd3bPNvfbjJ0ZroyJbuQu7C7p" + "669PRXdISTyC1TNv80pCbmWEJcaaCnk3TGksL5AXKw3UfOohiGOfxIOCHw9jHY1nxjeWD6Ww7sbrYOFAw0CJYIhe1jzDCnCF" + "tTE/mJv5e9yUWoQVgiFsFpk4TmwmCDQnrSnMEE+Jd6gPcLLf+M2MZTX4HLbTKA45aoB2XCuiT9ZT9KrGZOOm0d7MzZqxsUzh" + "VxChX+gGmkSO0q/SHMs72y3bQ8vfag+L23Y47GB4IIyF/hXSPWSibYxtrO2sLbstzXJWraO+VLyyJNcBlBgqD5BbSNuEUPqV" + "nKIvwVZa0f34CLrOh/BjLM50QcyYpY/TfIFFvl7e7p47npKemu5Ql+z43V4rfaq9rn2m470jxWHY+9gD6Rn2ac69rnoZ21yf" + "nE8dCY7Rjp7OEMc654yMyoFpbANgg799Yd5xwSn4jFLWFmodofwqrKbJQnO1ZsjQsCahf1vbK1ukW2J1MVY4KcwFO9kudMOD" + "zc6Qp8ELeAymGHGXPkILaJfN3XiesEDcKRSnPch0spdeEqNkopyXndJj0SL0IrPwc2yjOv1OP+EdbIveReun7zQT2F2WYTzT" + "OgVZ4FZwmE7NLhA5Nppe44KBWU/+FY0kPehPuoLaqIcspZWFdWKsfMyywXbEdkBdIbeVN1h+C8uIbBDVOqJ9SJp1sG1m2KjI" + "6KgGkRfCZodE2wxrrDVo/cP6wTrXGmrppNjhLKOEN0J7abTUWcT0JW/DnpqLWCLfxAexNvqXwFH/BH/dAA52CTz2j/HW8XQG" + "lLnD293bx1PD43V/gadmnuvuhxnRbu555Z3pu+BZ7lrveuh65on3fvTMz3DZ36UfcO7z3te7Glv09to37atxlX8lqXK0Jdq6" + "3XLOEms9YIsIuRA6I+x0+NDwAaHRNq5slffJR2Qmq1I+YQr6aAzQl+kB46d5wLip19RcgRrB6dpdoxl7xfbwKMRRbfwb/pPs" + "p5XFNGmjlCB2E24SCy1KI0FTC2lu3IntN44bb4x044QxXN+rXQxUDZQIJPsbBotqHfW7ekv9oD7COGpW4xfRX+QSXSh4BK9Q" + "XzwjSvI4Ja/6UPkgf5RqSM2lF1DlfJcaSYOkgdI0eZTUWxorFZEHKK3VDLWL5ara1nLU8tmaIyQyJNVW0NbaehckftzaDk7K" + "1dNKXXmrVFBKEqPEAO1FC5ICOMh/5cO4G07iN8eZd40YyIHVjdzGMWO1EW6YelP9qH5Hf6/XNsrqY7SZmk+rb9zTYoJ3/N99" + "Df2T/C8Dq7SPgak+X0bLjK/uEr6R/k3+Hd5ZGcmO2/aSzrYZZXzegKwtDxjeBb6+2p/sJCkF3nxTXCMa4i35rGVZSPbQ1qGX" + "bLr1hWWv2lTNrf6i3lZ+in+SQzzNzM5izR1mE7OF/jTo9yf6EwOdtK96DVMzC7IE8xY7juKJKnaUvksDZIu8Vm6jvFA2qyfU" + "HGpRpadUV4wWooU8wgTaj1RBSWyYed/IYVY0b5g5TMmw6o81ppc3j7E/+GP+FOJ1EFUnOWgd4bQ4X5okHYT4c0s8DDX1KDmn" + "PA+qyGNCUaGTUFvYQJNJKGSFEXgjqoVa8OmQOynqie6iUtiNp5Ph9BJUtyOl7vJ+ZaQ6Sa1guWIRrUnWN9bxttfW65aClli1" + "r+pRHsjT5ZJSM/EFrUQfkglkAN6B7vETPJ2/52/5CI54bV6GV0P90Xv0GvVG1RFFQV4V18VLcF4UZAvZUraJSXwcY6YXrOpc" + "cF+wmjZYuwvect230fvYG+W96W3stXuXeZO9g72jfdhnerODx2ieGH/LwIGA7o13L3ZJbp/ns39y8F6wTGCd76gvEDxi1kHh" + "uBtksHD8lZhCWfmJMkph8g95qDJFWaeUV+Yrk2SvtFE8S+/i+ugyyG8f97L3RiftRKBioFPwnvZOf6pP174G04O59QqmzNei" + "vWgMesGroI54EJ0LFv6LcJdmh5PXhXiiCDPpFvIJsmsRXodnwNkqsjNmS3Oi4dSHAzZYaNSGDH/Q/N2MMFeb81lfNA5bSDw+" + "iC/j4WQlHSLUESPF08JZ+gQQcZTAaFPhDBVpA9IJf4As3BNvwX6UyntwkS/lW/gD/pKf5PlQTkTQRJDtZfwDHyOlSVFSjvwk" + "u2gh+ZY63BqrtBRvCLklXW1giwm5Z30ia/QsLS19scyw7lW3i4/xKvQNmbSJ1EJsSbsjJ3sC2cxEGvbgDngef8Pqs3GsJMqJ" + "LbQVOYiK8aGsNHei4kSmS/BBHg8WWYZ15dPRH3yXWc7YqjXWFL2+cc7w6D+D4wO5/B/8CwNbg9eDawNvfMu8y71nfZ/8yYGQ" + "YJVAuH+dL+hL9r8OSNru4NZAG8Cgyb7K/iHglaavju+NZ61nmfeQ/0bgQ4AGogNnA5u0O0ZDbgHc4uQd0QXcX2gnXZRySylC" + "vBAQfpME+bX0u5hBF5I5+Djglt7oGy/MT5qTjIL6ccizTfUJxndjmPGrflOL0IdB/OsAqKY72sdX8UkgwdV0hDBAuEcdZB/J" + "DnKaKPygfahIOuI/0G20D61Cy3gO/sP8Zo40k81TZln203SZO808LIZl4624xsuiQ+gn6g54NgeZSl4Dzi1L29OF9A96gQ6m" + "PWkZWp6eIevJIjKQNCTpeAh+DzY1DC1Dh9F2NBhFoIN8B//ER6FkZMWv0Snwyc+oLt6N47EXNFyG5APMlRcyZRM8F88B9DUN" + "VUWf+Tn0lGymi2htfBAsoh3ZQI+LE0VVmItr4MWkgrhPOif+IswhGjbwG1oBcvAq6sFVcB2chseSAnQJ4fgFmouao79RNL6B" + "r+LO+BmvxPvBiS5Bj4pdfAnbY3KzIH/Bz4NdTDaHGSWMQ8ZaswfrYYYZT7VnWjc9ylhotDEW6wO1O5AZLmiadlcrpcnatuCc" + "YKvg8uCtYB1tp7Zfq6sVB0RcWOurubRhejP9kvY4GAjm1dZoKcHdwa+B9oE431jfLX/rYDDYJZgW6BS4F6ivXTR6A25dzXez" + "KrwsjhS+iYa0X7oIOL28RJQNUJ8fl03psLhfKCu8A83NJs/xO/SUt2A1TQm8DvKfqZjTjRCjqNHT/MEOg9ev5PNA3m9REO8E" + "NFBHWAzzFtDFQmnxunhPdAoXaCnamJ6morALPH0XyLcxmoI+ITvUAU4+lecDC1vMG6NnaD9YksZrQ3XQAktkA7lOupAK5DeC" + "6T+0uTBCoLBCY1qf7qC1AIlFCvkA2dwFNC2CHTzG1fFKpPNlwM02fpd7uJcfBPlrrBTE0rloCrlLCN2FV+MreAf9IVSVTHGU" + "aNI0ekEoJFWXR0itxLpCPL0D9b5T2C10oJOIieuQaWQumUU+YahV0W9oLWIoDb1EfVB+tIB35Im8NVqCBiEVjeBNAft/56Mh" + "lqSDN3xjn1kDfhsqkY38LcvDyrOS7G+WDFSUXTTzmQ3NoeYnM948aYabF4zZxjJjr+ExrhlrjMZGFcOlNzfyGtkhC/cGSRcy" + "DF3XX+g39WqGYUjmPWORscAoYqwyJhjbjQ+Aek9BpXkCaiMXRH9Zp8ZnvYeeDlXSDn2LKfEcqCiKBT4eo5NkidBa7CEmCYOE" + "CUJZMZdUVfoilhJLCoOpi1yCk4aSebgfeKmfLWBlWS3WkxVgDyDmTjefm9HMyabwhXw7j+V+QNeHcQhpCxIqC1X3JFIeasZH" + "oOupJCdxQUSeDh6ajNvh2agMYmDzSZDVXvO2/ALgjHdmPkZ5V96Mn4PqcxeL5PGQFRag0Wge1K+FoaJ9g29BDLgD9ZJK+pGl" + "pDv47Rt8AE/HM3B9XBG/RVsgOiXye4ATN4D91AWPSmSPGeEvWSrLz/vw8lCZKVCpreU/uZ1f5M/5bIjLR8FXn5G+OD9+hr/R" + "cDFB/CKMoflJFdpPWC01kz4Jg+hkUonE0F1CmnCbRpGf6ADEj1nATUWcG5Xgw1lr9hL2ymCbWV7GjarGF+Oc2Zc1YGvNw0Zl" + "o7MRajrNtuyC6TdqGaVAQwUg0iWZVcwWRub/kSpnjDBaGYn6bEBWrfWaemn9ofZKOwTe30JbrH3VtukrdFnvpaUHPwYfapHG" + "buO23l+bERS1QfpOI8FoYVzRNgUnB4fq1c3b5gTDDk8tgh20WP2AEWkc0ioHjwRqB84ECgdzabX0aVBHm8FhwU3BNvpHcw/4" + "wmIWYdYyv7OLWBDaCXPJSbCNENyZVhIThLx0HqrH/+RV8ESSg0xGvVl1UzZfQ4Um8pHsF7OYEdSnGtPMOnD2juwzVO5fzSfs" + "B98OmX0haohKodVoEG5F/oQa+ig2UDbcEGrwAmQcXgQV8Vq+HuJFgJ/mRfmvLNlMNXuy2+wHm8/c5iPTY66ASroxj+ClOGdN" + "wFIqo2gUwKWon06k+0k0vSdYAH80VprLJQBhF5WvKbvUi8ouaZyYU2wAttxYGiEuojvxLaizk7CJo/FRHsZGm6/MBawo38QO" + "md+N341NRjtTZArbbJrGTyPJKGHOMLeaG8w+ZoaxzmhvdDfyg34GgCY362+1G1qaVhF0lK7N1eppWCukjdUua7u07oCE/9Yu" + "aY+04lB7RulYb6iv1f36Ez1WZxDF8xtljUHGTqODEWF81csY0fB8yzhvtATNn9bP68eMp8ZRw69P1wfoY2BORV3TdmvttSda" + "DtD/St2mH9WG6E0MK1TFPVgFVpltZFd5PDqJd+D7+DTW8ABikmq0CN1EOpM+pAy5jffjXTgP3gY5qAh6D9k0yA6yADvPtrAB" + "rB3ryiaxdNYKUGVR3gAiZHPIyHlxBdwCb8Yy+YMECKI7yQVyk1Sko2lniml7cgnk2BM8OTeJxhPAj1uDd37OnIcW8t58HQ/y" + "pqgGikQPOEJ7kIBDwC/forKQM7dDbGgC8fsFROcA3g94oL+gizHSXvEBDRO4sEQ+qvRRo5RYQFfHxXrKR7WKmiAFhexCTaG/" + "9Ctg+TNCfTIMFyHT6Sc6nwZxKJoBEaQmPoXjkIx+5QIgiWdZ/z9tOrcD6nrKPjAXa80L8sesGgsCpkgzK7LZLJrNM+1GqpHd" + "3GdeNqeapc32Rro+3GgMz83NP4yuoPPFRnmwv1um1exnLDGY8c58b640J5oPjUuGag4yR5uFoAYZZa6A3wwjxGxh7jYlFsZe" + "mH+aE0wCWPU31oodBQ4UvoddAzk35GtAxicgwiWznFAfVASMUZUfYe1ZDXYKeJ3F3piNzApmN/O8mWJeMTuayGxjPgZLvccO" + "sBOsMh8Lc74AbmwNES832Uzy0xi6mt6nNkDN+4UzQpywWTgEGNovHBGWC4uEacIGYYXQWsgvvIEs5qTRQlXhDh0A6GkT/Uwj" + "hDDhHO1LR9A9lIKMswkfICcvAmT1lHrobcjZeagVcFs8OUQWkpLkOF6J/8Br8C84ATDUTPQraomaoCoomZ+BU+3kY/gQPpoP" + "4Pn5a+ZmYZBn4yD6XmYzWQfWh/0JVrcSbG4Q6wTeHcdk7mGxEOcNFs3nQBwewpvwipBHdJ4TneLt+Ax+E3Lpab6cl+aJIL/T" + "7B82mjnN2SCjbmZuwBxTjOZGTfCmTB/abQwxphnfjJVQHeaFODLf3GN2gsxaAmTdgSUCqlzDMmP/FPan2d8caNrYNjaDOSBa" + "5THXmB3YWeBvBLsPntWTPWCD4BTPYN5A0GIhfoyf4934PqaZ7VhuyC4ePoEfg3eH2G98EFqBHMD9L5BNjqJeeBXuiL+gE6g5" + "5KjF5CHIrT5g0t+JQAfR6bQLIJVcgFcH0aX0OX0GuptBp9FdoEFF8NFXUKXsAVzzmJo0lxAuIEDK72gcdVMXzSOUFpLpXnqc" + "vqcBKgspNJ4mgo7O0GP0Mj0H1610Dm1NI6lB/iEXyRrItDlJCkSFvoCCnOgaZNHeKDs6C3KtDNXObfYXG8PysX/MhWZb8zez" + "u1nHDBiPjT+NbXANMweDV3SGqjjaXG5WAinMYb8wysLheoF9Z59YP3bHRCChC2wR42Cru83rUEHbANEkmTpg+UKAbxLN/FDn" + "bGI3IDdeYW1YX9DgHL4L9D6DTQdrqMj/gHqrFlSqMvjqKf6RL+CxLDfkkO6sC8g0lV0xx5onzPqgncfA7XKzgfnSfAkYys2G" + "slxgSRfZYO7ml3l2/gkqqFjI6REoFOLcfrivgOYDTUTd0Ep0ErBlFTwI98R9cCwOJ+1ISzIOsA0nrcAb9tBVdBlIuq1wVYgR" + "GgrFhYrCDuG94BO2CNWFdFpDOCDsgTc24Ss9CXrjtKTwnY6lw0GjW0CTo0GfPWgL2px2h5o9B5UgTn4mxyBG9iZNSSo+D1G4" + "J7ZAvmuOCiGM4gCbNOGP2E6wycxqPYaVY5/MJ6YXrLQAE1kEK8JGwpt1bDf7BtF5KoxvxkuAbxREf6HJyAOy6QxW+BdKRfkQ" + "Bvm9ZHY+BCLgHrbRPGyeZD7+hhfgCeZ4sybrwXsgO7/BcrLGbDHvDRK5xW+x9Swc8Noy9JaH8f3sKqvAu/OZPIT3Y7VZL5bE" + "7oDH5mdW0Fpb8N+hrAlzmcUga5wDPxwPmf4j+MV4sKUVIPvi/A5vC7IuDRX/Rn4B4v8OyP3loHpdwHOh8+g71HgR6AO3gh4a" + "44E4HY0EnRTEx7AbqqKWuDTuhr/g/qQu8eG1gPFU0gFQZyNSk+g4P0S5O+Qc5LPGgPoXkPPQJpI2gP0WgH/NIc2BOoB/7YAZ" + "MaQaaUbGQN5aQlqRPKQErNiF1CGYnMDH8Vf8Cf+N22IRJ0EV2BgXww4UA/KrApzfBn7qgd084gSeqwO37yDuzIOYdgEs9A6c" + "sy54+C98OOS3pTyGV+HR0NOfb+Vn+QNAv1P5Zs4ArRZBtVBhVAxVg6ozFs6dAtcZaCj6A71AOXADQD/zwQZi0D9QzXaAKDoD" + "0FFtFI/64SlYwNXQd94eEdwf94J3ZaFyqYycaDzOh7cgBSE0AOJKdvwTdUZfuJN3hrwbD/VtEcTBBzqDhb8EVDoD6qVBaCfS" + "kIyj8DW0FWT+DZXANjivHR0DPrzoF1wV18YI5JAT+gfgybg3rgnv2wFPeyBLL4GePhDJNuFrED+24EOABg7B3TnQzVl4vxIw" + "xd+AfXeCL+2E380wFmo4sPIL+B6+iY/i61B7fcXpMP4ESP0baPcFfgn19k5A6/vwZbwY/4qbAw4cj3/HkwBDExyGa+HfQEIN" + "sQR6aYy742kQSfNhDjh7CnjuQOBPxSWhuh6AO8HJ7qPHyAZzauMfYGFX4akYrgZzn0DN74c5Q2F9N5xVwiNhz93AcyWwtNX4" + "FQ7CGWbiIVBP3McZ2AOn7I1H47v4LT6Jx+GyIIVecP4FuD2OAC1UwIPh7S+YAo/FYeQ6OPkUsNhieAR+CLXINuCtLm4C1xP4" + "COyVDeuoAIxsB/uXwCnoOHoEVpADZP8CUNAZdAc9QXHIA56bBHyGg9YzUC7cFFaoB3G6GlBFeOoBZy8Ke/qRgq1YQ270APR4" + "Al1Cr9EhqHJmgkedh7sD4Ge7AGUdQgkoACgsDr1D6VmaL4Jz47xYhhq1DkgtJ1h9KPSaIJGnMOYruoXuocuQxcehsWAzM9Bc" + "NAZNh+sk8M3xsGLm10br0Z/oAljwKRj7Cj2E2BMLfFyD5+3ob5h9Aew4HnoWoo5gZ3HAwQPwoxhA6KPA0r6g56g9VHkR8HQQ" + "bHUVaoUk8KwY4Pwi6oWyoVyoDRoIz0MAL3ohducCPFEc+fg/PJVbsr7LskFzAJ7rhPqiquAPAioP/tMcVYZxAYhxGbw8zHXy" + "q4BAbvJX/BvEtDi+B/LzTOi5Dv47iQ/iXSDa9uUT+TTeizeCyDkK3o4E/LUavPYoP86vwHUt5KVrsO9tvgXmDOBzwdO381kw" + "PhQwc33o6Qy45ht7zt6yCMApeXgG5P9tkJdUHgkY4x92GCiOBSGuMvYOMtMz5oPa2M88gPzesIcQo++xF5AHr8P1NvzugAz4" + "AOgji4fK8SWsYGcCz8utXIV6pwjE+/K8NuxUCCJsLdi9HUScXLBvDog+9Xh96E/KmpON5wSk9Ra4OQVR28841Lqn2XZATKks" + "FNZKhVG3YQcJcFQYd7LPEN3DYJ98gJZdgNMMVpiXhCedpQDHCE4j8gCMT8j6gkmHmvYDcPg16ykF8lECy/wXZD9gqcx/S9Zh" + "fjjwLMFqHDCdwA0WAs0LY3U4T6YMMr+I4jDHBOlk7v+NMajJ3bDmOzj9VxhpByl9YjdBMq9h/ZfsFrsE0jnFjgBCPMk2sFWA" + "HLextWw5mwd36yB7zQe0PIdNAyy/AbLRQshQw9kU6N/H/mDLoKoZAW0zzN4I2Ws0GwLzFrAlMHoYIJe+bCqMmQH9fWHWDMjA" + "2wCH7oCxu2DXJ+wV6O4+aOwVcPMJatFPwNFd6HsE7Snc3QVOrwCufQTvnwLPz2HOY/ae/QSJp8GJ3kCzg4R0kIcGJwsyK+gw" + "G2iVQZ+Y9YWZyhFPgLVTs74ZSwRJPIP93rMvgFlT4PoeUFYCzMWcgEw1kLKPabCeAjoUuBfeuOGdBTK2BTSX+RUagvUj4UmC" + "0UF4p4AeEcjfAfIPQnOBTtJhxwSgL6DTz7DHBzhB5lnuw1nuAd1kf4PMTwFie5aF2U6BPG7AGd8D3YOnk5D/n8Cch9B/HGqb" + "83DuNzD7HNh+LEjkJTxfh/7tgFxOgA7Psr2AGdYB0jkNY87CmL9g5EUY9Qxs9Ro7A5b5AlZ4AatfgBkXofch+ETmzifg/Y0s" + "zu5D73XofZxlH09h/AewoR8gtUzJpcPZvf/5WVrWd3SMUZCaAXWuASdnIBkdbA+BNEx4MqE/CM0DErHDDCfc+0G6Bsz8l4ws" + "+8xcReaZI3WwXpYlvUSQpi9rTS9I8SM0b9a85CzNpWRpiAK+/Q7c2WFU5peCOoxxQKOgEQY7uWCcEzTmyfKLn2BdicBFpgc4" + "4ZoEazmybMYH751gHQ6Yk8m9BnNSss6dBL0JWfvHAb2DnTNl8SVrpWSYw4CLf79htIK1RPIoiFyleTGoggtlURmIKqXA54tA" + "1V8BYkwT3hjiSQ2oResDzmzD2/JWvDn8doZ4GcMH8t+zrkP5CIiM0wHrTAGaDL9zAN8tgjYPouVivgqq240QRxfyJXC/EbDR" + "Zoihq/kGiMKxgJNO8YOAy49CXL4FNdRtoJtQDb/g7wD3v4Xfp0AvIYa/4E/+o8fQnkM9/5rH8w8Q239Cre+CPGEHvOOG2O/M" + "ujqA0uDXBe8/8s8w7itP4Mk8Ba6JWTM8PIn/gJYIfZlfun6AFd/Avp8z/44aKB5mpMCoVOj5B3Z9Du/igJeHwN8d4OE1PL3L" + "4vMb7O3nBmA9g5tQqQazyOAUWSFXyUgEnGaDKisfygF1iIwsKArlRjmhT0SZowXokSEneoBnDZ4yv0PWss4QhDU1HgB+0+HZ" + "CyfL/NtqB/xqWd/oemBfjWd+s4wgQ2a+C3IF8imGJy/wYoV9MneKhL0iUQHA4uWBiqKCgDCrZH2pXAdQaUVUKeu75frQakNG" + "rQitMqoAWLMkUGkYWzyLMr91LoTyojyQo3PBunlh5QhYNxKFoXDYQYLz/PvVNAb+vf/x6wPOU7Iknwb3buA+Fe7c8NYP5M3i" + "WoNz/vsHAW4lsIqUJbd/v8KWQH4yXKX/VpeyrlAI//elNs36Jf/d/89vuDMl+b+/7/73G+/MHTL34P9jv/97/ffd/3369/d/" + "AYxHlHJ2PgAA" +) + + +@lru_cache(maxsize=1) +def ready_trigger_pcm() -> bytes: + return gzip.decompress(base64.b64decode(READY_TRIGGER_PCM_16KHZ_MONO_GZIP_B64)) diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index f5aa600ab84..85ff7fcbf7d 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -60,7 +60,7 @@ "limit": 4 }, "BLE001": { - "limit": 2904 + "limit": 2903 }, "C401": { "limit": 11 @@ -255,7 +255,7 @@ "limit": 480 }, "S110": { - "limit": 237 + "limit": 236 }, "S112": { "limit": 24 diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py new file mode 100644 index 00000000000..18e48169bc1 --- /dev/null +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -0,0 +1,174 @@ +import json +import os +import sys +import types +from unittest.mock import MagicMock + +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) # Adds the parent directory to the system path + +from litellm.llms.bedrock.realtime.handler import BedrockRealtime +from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig + + +class FakePayloadPart: + def __init__(self, bytes_): + self.bytes_ = bytes_ + + +class FakeInputChunk: + def __init__(self, value): + self.value = value + + +class FakeInputStream: + def __init__(self): + self.sent = [] + self.closed = False + + async def send(self, event): + self.sent.append(event) + + async def close(self): + self.closed = True + + +class SendFailingInputStream(FakeInputStream): + async def send(self, event): + raise RuntimeError("bedrock send failed") + + +class FailOnPromptEndStream(FakeInputStream): + async def send(self, event): + payload = json.loads(event.value.bytes_.decode("utf-8")) + if "promptEnd" in payload.get("event", {}): + raise RuntimeError("bedrock rejected promptEnd") + self.sent.append(event) + + +class FakeBedrockStream: + def __init__(self, input_stream=None): + self.input_stream = input_stream if input_stream is not None else FakeInputStream() + + +class DisconnectingClientWS: + def __init__(self, messages): + self._messages = list(messages) + + async def receive_text(self): + if self._messages: + return self._messages.pop(0) + raise RuntimeError("client disconnected") + + +class ClosableClientWS: + def __init__(self): + self.closed = False + + async def close(self): + self.closed = True + + +class EndedBedrockReceiver: + async def receive(self): + return None + + +class EndedBedrockStream: + async def await_output(self): + return (None, EndedBedrockReceiver()) + + +@pytest.fixture +def stub_aws_models(monkeypatch): + package = types.ModuleType("aws_sdk_bedrock_runtime") + models = types.ModuleType("aws_sdk_bedrock_runtime.models") + models.BidirectionalInputPayloadPart = FakePayloadPart + models.InvokeModelWithBidirectionalStreamInputChunk = FakeInputChunk + package.models = models + monkeypatch.setitem(sys.modules, "aws_sdk_bedrock_runtime", package) + monkeypatch.setitem(sys.modules, "aws_sdk_bedrock_runtime.models", models) + + +class TestBedrockRealtimeHandler: + """Client disconnect must close the Bedrock session gracefully (LIT-2239 regression)""" + + @pytest.mark.asyncio + async def test_client_disconnect_flushes_session_close_messages(self, stub_aws_models): + handler = BedrockRealtime() + config = BedrockRealtimeConfig() + stream = FakeBedrockStream() + client_ws = DisconnectingClientWS( + [json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})] + ) + + await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + + sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] + event_names = [next(iter(event["event"])) for event in sent_events] + assert event_names[0] == "sessionStart" + assert event_names[-2:] == ["promptEnd", "sessionEnd"] + assert stream.input_stream.closed + + @pytest.mark.asyncio + async def test_client_disconnect_before_session_update_sends_nothing(self, stub_aws_models): + handler = BedrockRealtime() + config = BedrockRealtimeConfig() + stream = FakeBedrockStream() + + await handler._forward_client_to_bedrock( + DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {} + ) + + assert stream.input_stream.sent == [] + assert stream.input_stream.closed + + @pytest.mark.asyncio + async def test_input_stream_closed_even_when_close_flush_fails(self, stub_aws_models): + handler = BedrockRealtime() + config = BedrockRealtimeConfig() + stream = FakeBedrockStream(input_stream=SendFailingInputStream()) + client_ws = DisconnectingClientWS( + [json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})] + ) + + await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + + assert stream.input_stream.closed + + @pytest.mark.asyncio + async def test_close_flush_continues_after_partial_send_failure(self, stub_aws_models): + handler = BedrockRealtime() + config = BedrockRealtimeConfig() + stream = FakeBedrockStream(input_stream=FailOnPromptEndStream()) + client_ws = DisconnectingClientWS( + [json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})] + ) + + await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + + sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] + event_names = [next(iter(event["event"])) for event in sent_events] + assert "sessionEnd" in event_names + assert stream.input_stream.closed + + @pytest.mark.asyncio + async def test_bedrock_stream_end_closes_client_websocket(self): + handler = BedrockRealtime() + client_ws = ClosableClientWS() + + await handler._forward_bedrock_to_client( + EndedBedrockStream(), + client_ws, + BedrockRealtimeConfig(), + "amazon.nova-sonic-v1:0", + MagicMock(), + {}, + ) + + assert client_ws.closed + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py index bf15727f4b4..a68aa603b26 100644 --- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py @@ -5,11 +5,16 @@ from unittest.mock import MagicMock import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path +sys.path.insert(0, os.path.abspath("../../../../..")) # Adds the parent directory to the system path -from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig +import base64 + +from litellm.llms.bedrock.realtime.transformation import ( + TRIGGER_LEADING_SILENCE, + TRIGGER_TRAILING_SILENCE, + BedrockRealtimeConfig, +) +from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm from litellm.types.llms.openai import OpenAIRealtimeEventTypes @@ -67,19 +72,14 @@ class TestBedrockRealtimeConfig: } ] - session_config = config.session_configuration_request( - "amazon.nova-sonic-v1:0", tools=tools - ) + session_config = config.session_configuration_request("amazon.nova-sonic-v1:0", tools=tools) session_dict = json.loads(session_config) prompt_start = session_dict["prompt_start"]["event"]["promptStart"] assert "toolConfiguration" in prompt_start assert "tools" in prompt_start["toolConfiguration"] assert len(prompt_start["toolConfiguration"]["tools"]) == 1 - assert ( - prompt_start["toolConfiguration"]["tools"][0]["toolSpec"]["name"] - == "get_weather" - ) + assert prompt_start["toolConfiguration"]["tools"][0]["toolSpec"]["name"] == "get_weather" def test_transform_tools_to_bedrock_format(self): """Test OpenAI tool format to Bedrock format transformation""" @@ -93,9 +93,7 @@ class TestBedrockRealtimeConfig: "description": "Get current weather", "parameters": { "type": "object", - "properties": { - "location": {"type": "string", "description": "City name"} - }, + "properties": {"location": {"type": "string", "description": "City name"}}, "required": ["location"], }, }, @@ -120,18 +118,11 @@ class TestBedrockRealtimeConfig: # Test PCM16 format assert config._map_audio_format_to_sample_rate("pcm16", is_output=True) == 24000 - assert ( - config._map_audio_format_to_sample_rate("pcm16", is_output=False) == 16000 - ) + assert config._map_audio_format_to_sample_rate("pcm16", is_output=False) == 16000 # Test G.711 formats - assert ( - config._map_audio_format_to_sample_rate("g711_ulaw", is_output=True) == 8000 - ) - assert ( - config._map_audio_format_to_sample_rate("g711_alaw", is_output=False) - == 8000 - ) + assert config._map_audio_format_to_sample_rate("g711_ulaw", is_output=True) == 8000 + assert config._map_audio_format_to_sample_rate("g711_alaw", is_output=False) == 8000 def test_transform_session_update_event(self): """Test session.update event transformation""" @@ -158,12 +149,7 @@ class TestBedrockRealtimeConfig: # Verify session start message session_start = json.loads(messages[0]) - assert ( - session_start["event"]["sessionStart"]["inferenceConfiguration"][ - "temperature" - ] - == 0.9 - ) + assert session_start["event"]["sessionStart"]["inferenceConfiguration"]["temperature"] == 0.9 def test_transform_session_update_with_tools(self): """Test session.update with tools""" @@ -237,12 +223,7 @@ class TestBedrockRealtimeConfig: content_start = json.loads(messages[0]) assert content_start["event"]["contentStart"]["type"] == "TOOL" assert content_start["event"]["contentStart"]["role"] == "TOOL" - assert ( - content_start["event"]["contentStart"]["toolResultInputConfiguration"][ - "toolUseId" - ] - == "call_123" - ) + assert content_start["event"]["contentStart"]["toolResultInputConfiguration"]["toolUseId"] == "call_123" def test_transform_input_audio_buffer_append(self): """Test input_audio_buffer.append transformation""" @@ -260,12 +241,7 @@ class TestBedrockRealtimeConfig: content_start = json.loads(messages[0]) assert content_start["event"]["contentStart"]["type"] == "AUDIO" - assert ( - content_start["event"]["contentStart"]["audioInputConfiguration"][ - "sampleRateHertz" - ] - == 16000 - ) + assert content_start["event"]["contentStart"]["audioInputConfiguration"]["sampleRateHertz"] == 16000 audio_input = json.loads(messages[1]) assert audio_input["event"]["audioInput"]["content"] == "base64_audio_data_here" @@ -286,6 +262,144 @@ class TestBedrockRealtimeConfig: assert "contentEnd" in content_end["event"] +class TestBedrockRealtimeResponseCreate: + """response.create must trigger Nova Sonic generation (LIT-2239 regression)""" + + def _start_session(self, config): + config.transform_realtime_request( + json.dumps( + { + "type": "session.update", + "session": {"instructions": "You are a helpful assistant."}, + } + ), + "amazon.nova-sonic-v1:0", + ) + + def test_response_create_before_session_update_is_noop(self): + config = BedrockRealtimeConfig() + + messages = config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0") + + assert messages == [] + + def test_response_create_emits_spoken_trigger_audio(self): + config = BedrockRealtimeConfig() + self._start_session(config) + + messages = config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0") + + assert len(messages) > 1 + + content_start = json.loads(messages[0])["event"]["contentStart"] + assert content_start["promptName"] == config.prompt_name + assert content_start["contentName"] == config.audio_content_name + assert content_start["type"] == "AUDIO" + assert content_start["interactive"] is True + assert content_start["role"] == "USER" + assert content_start["audioInputConfiguration"]["sampleRateHertz"] == 16000 + + audio_events = [json.loads(message)["event"]["audioInput"] for message in messages[1:]] + assert all(event["promptName"] == config.prompt_name for event in audio_events) + assert all(event["contentName"] == config.audio_content_name for event in audio_events) + + sent_pcm = b"".join(base64.b64decode(event["content"]) for event in audio_events) + assert sent_pcm == TRIGGER_LEADING_SILENCE + ready_trigger_pcm() + TRIGGER_TRAILING_SILENCE + + def test_second_response_create_reuses_open_audio_content(self): + config = BedrockRealtimeConfig() + self._start_session(config) + + first = config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0") + second = config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0") + + assert len(second) == len(first) - 1 + assert all("audioInput" in json.loads(message)["event"] for message in second) + + def test_response_create_is_noop_when_client_streams_audio(self): + config = BedrockRealtimeConfig() + self._start_session(config) + config.transform_realtime_request( + json.dumps({"type": "input_audio_buffer.append", "audio": "c2lsZW5jZQ=="}), + "amazon.nova-sonic-v1:0", + ) + + messages = config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0") + + assert messages == [] + + def test_client_audio_after_trigger_reopens_block_at_client_sample_rate(self): + config = BedrockRealtimeConfig() + config.transform_realtime_request( + json.dumps( + { + "type": "session.update", + "session": { + "instructions": "You are a helpful assistant.", + "input_audio_format": "g711_ulaw", + }, + } + ), + "amazon.nova-sonic-v1:0", + ) + config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0") + trigger_content_name = config.audio_content_name + + messages = config.transform_realtime_request( + json.dumps({"type": "input_audio_buffer.append", "audio": "c2lsZW5jZQ=="}), + "amazon.nova-sonic-v1:0", + ) + + events = [json.loads(message)["event"] for message in messages] + assert [next(iter(event)) for event in events] == [ + "contentEnd", + "contentStart", + "audioInput", + ] + assert events[0]["contentEnd"]["contentName"] == trigger_content_name + new_content_start = events[1]["contentStart"] + assert new_content_start["contentName"] == config.audio_content_name + assert new_content_start["contentName"] != trigger_content_name + assert new_content_start["audioInputConfiguration"]["sampleRateHertz"] == 8000 + assert events[2]["audioInput"]["contentName"] == config.audio_content_name + + def test_client_audio_after_trigger_reuses_block_at_matching_sample_rate(self): + config = BedrockRealtimeConfig() + self._start_session(config) + config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0") + trigger_content_name = config.audio_content_name + + messages = config.transform_realtime_request( + json.dumps({"type": "input_audio_buffer.append", "audio": "c2lsZW5jZQ=="}), + "amazon.nova-sonic-v1:0", + ) + + assert len(messages) == 1 + audio_input = json.loads(messages[0])["event"]["audioInput"] + assert audio_input["contentName"] == trigger_content_name + + def test_session_close_messages_close_audio_prompt_and_session(self): + config = BedrockRealtimeConfig() + self._start_session(config) + config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0") + + close_messages = [json.loads(message)["event"] for message in config.session_close_messages()] + + assert [next(iter(event)) for event in close_messages] == [ + "contentEnd", + "promptEnd", + "sessionEnd", + ] + assert close_messages[0]["contentEnd"]["contentName"] == config.audio_content_name + assert close_messages[1]["promptEnd"]["promptName"] == config.prompt_name + assert config.session_close_messages() == [] + + def test_session_close_messages_before_session_update_is_empty(self): + config = BedrockRealtimeConfig() + + assert config.session_close_messages() == [] + + class TestBedrockRealtimeResponseTransformation: """Test suite for response transformation""" @@ -296,11 +410,7 @@ class TestBedrockRealtimeResponseTransformation: logging_obj.litellm_trace_id = "trace_123" bedrock_message = { - "event": { - "sessionStart": { - "inferenceConfiguration": {"maxTokens": 1024, "temperature": 0.7} - } - } + "event": {"sessionStart": {"inferenceConfiguration": {"maxTokens": 1024, "temperature": 0.7}}} } result = config.transform_realtime_response( @@ -330,9 +440,7 @@ class TestBedrockRealtimeResponseTransformation: logging_obj.litellm_trace_id = "trace_123" # First create a content start to initialize IDs - content_start_message = { - "event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}} - } + content_start_message = {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}} result1 = config.transform_realtime_response( json.dumps(content_start_message), @@ -368,9 +476,7 @@ class TestBedrockRealtimeResponseTransformation: ) # Check for text delta - text_deltas = [ - msg for msg in result2["response"] if msg["type"] == "response.text.delta" - ] + text_deltas = [msg for msg in result2["response"] if msg["type"] == "response.text.delta"] assert len(text_deltas) == 1 assert text_deltas[0]["delta"] == "Hello, world!" @@ -384,9 +490,7 @@ class TestBedrockRealtimeResponseTransformation: logging_obj.litellm_trace_id = "trace_123" # First create a content start for audio - content_start_message = { - "event": {"contentStart": {"role": "ASSISTANT", "type": "AUDIO"}} - } + content_start_message = {"event": {"contentStart": {"role": "ASSISTANT", "type": "AUDIO"}}} result1 = config.transform_realtime_response( json.dumps(content_start_message), @@ -404,9 +508,7 @@ class TestBedrockRealtimeResponseTransformation: ) # Now send audio output - audio_output_message = { - "event": {"audioOutput": {"content": "base64_audio_content"}} - } + audio_output_message = {"event": {"audioOutput": {"content": "base64_audio_content"}}} result2 = config.transform_realtime_response( json.dumps(audio_output_message), @@ -424,9 +526,7 @@ class TestBedrockRealtimeResponseTransformation: ) # Check for audio delta - audio_deltas = [ - msg for msg in result2["response"] if msg["type"] == "response.audio.delta" - ] + audio_deltas = [msg for msg in result2["response"] if msg["type"] == "response.audio.delta"] assert len(audio_deltas) == 1 assert audio_deltas[0]["delta"] == "base64_audio_content" @@ -504,14 +604,67 @@ class TestBedrockRealtimeResponseTransformation: # Should have text.done, content_part.done, and output_item.done assert len(result["response"]) == 3 - text_done = [ - msg for msg in result["response"] if msg["type"] == "response.text.done" - ][0] + text_done = [msg for msg in result["response"] if msg["type"] == "response.text.done"][0] assert text_done["text"] == "Hello, world!" # Delta chunks should be reset assert result["current_delta_chunks"] is None + def test_content_end_end_turn_emits_response_done(self): + """END_TURN contentEnd must produce response.done (LIT-2239 regression)""" + config = BedrockRealtimeConfig() + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_123" + + content_end_message = {"event": {"contentEnd": {"stopReason": "END_TURN", "type": "AUDIO"}}} + + result = config.transform_realtime_response( + json.dumps(content_end_message), + "amazon.nova-sonic-v1:0", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": json.dumps({"configured": True}), + "current_output_item_id": "item_123", + "current_response_id": "resp_123", + "current_conversation_id": "conv_123", + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": "audio", + }, + ) + + response_done_events = [msg for msg in result["response"] if msg["type"] == "response.done"] + assert len(response_done_events) == 1 + assert response_done_events[0]["response"]["status"] == "completed" + assert result["current_output_item_id"] is None + assert result["current_response_id"] is None + assert result["current_delta_type"] is None + + def test_content_end_partial_turn_does_not_emit_response_done(self): + config = BedrockRealtimeConfig() + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_123" + + content_end_message = {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN", "type": "TEXT"}}} + + result = config.transform_realtime_response( + json.dumps(content_end_message), + "amazon.nova-sonic-v1:0", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": json.dumps({"configured": True}), + "current_output_item_id": "item_123", + "current_response_id": "resp_123", + "current_conversation_id": "conv_123", + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": "text", + }, + ) + + assert all(msg["type"] != "response.done" for msg in result["response"]) + assert result["current_response_id"] == "resp_123" + def test_transform_prompt_end_response(self): """Test promptEnd response transformation""" config = BedrockRealtimeConfig() @@ -552,9 +705,7 @@ class TestBedrockRealtimeResponseTransformation: logging_obj.litellm_trace_id = "trace_123" # Create a sequence of messages - content_start = { - "event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}} - } + content_start = {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}} text_output1 = {"event": {"textOutput": {"content": "Hello"}}} text_output2 = {"event": {"textOutput": {"content": " world"}}} @@ -600,9 +751,7 @@ class TestBedrockRealtimeResponseTransformation: logging_obj.litellm_trace_id = "trace_123" # Create a sequence of messages - content_start = { - "event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}} - } + content_start = {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}} text_output = {"event": {"textOutput": {"content": "Hello"}}} all_events = [] @@ -636,9 +785,7 @@ class TestBedrockRealtimeResponseTransformation: ) # Check all response_ids are the same - response_ids = [ - event["response_id"] for event in all_events if "response_id" in event - ] + response_ids = [event["response_id"] for event in all_events if "response_id" in event] assert len(set(response_ids)) == 1, "Response IDs should be consistent" From 2a9dbc4c0d2673ee50469229fff60776ceb68c1f Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 1 Jul 2026 20:13:58 -0700 Subject: [PATCH 06/18] chore(ui): remove unused dep, delete dead file, and unblock knip (#31933) Knip flagged remark-gfm as unused and date-fns as imported-but-undeclared, so drop remark-gfm (which prunes its transitive markdown subtree from the lockfile) and declare date-fns, which keyExpiryUtils.ts imports but only received transitively. Also delete the dead memory/components/index.tsx barrel, since nothing imports it once the page pulls MemoryView from its module directly Knip itself could not run: its Playwright plugin imports every config referenced by a --config flag in package.json scripts, and migration.serverRootPath.config.ts threw at import time when SERVER_ROOT_PATH was unset. Move that guard into a config-specific globalSetup so importing the config is side-effect-free; the check still fires loudly before any test runs when the prefix is missing --- .../migration.serverRootPath.config.ts | 10 +- .../migration.serverRootPath.globalSetup.ts | 12 + ui/litellm-dashboard/package-lock.json | 295 +----------------- ui/litellm-dashboard/package.json | 2 +- .../(dashboard)/memory/components/index.tsx | 1 - 5 files changed, 15 insertions(+), 305 deletions(-) create mode 100644 ui/litellm-dashboard/e2e_tests/migration.serverRootPath.globalSetup.ts delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/memory/components/index.tsx diff --git a/ui/litellm-dashboard/e2e_tests/migration.serverRootPath.config.ts b/ui/litellm-dashboard/e2e_tests/migration.serverRootPath.config.ts index 205348463c8..d32f59b16bf 100644 --- a/ui/litellm-dashboard/e2e_tests/migration.serverRootPath.config.ts +++ b/ui/litellm-dashboard/e2e_tests/migration.serverRootPath.config.ts @@ -6,14 +6,6 @@ import { defineConfig, devices } from "@playwright/test"; * running. globalSetup logs in at `${SERVER_ROOT_PATH}/ui/login` so the admin * storage state is valid under the prefix. */ -if (!process.env.SERVER_ROOT_PATH) { - throw new Error( - "migration.serverRootPath.config.ts requires SERVER_ROOT_PATH to be set (e.g. SERVER_ROOT_PATH=/litellm). " + - "Without it this config silently re-runs the default mount and never exercises the prefix. " + - "For the root-less run use the default playwright.config.ts (npm run e2e:migration).", - ); -} - export default defineConfig({ testDir: "./tests/migration", testMatch: ["migratedPages.spec.ts"], @@ -34,5 +26,5 @@ export default defineConfig({ projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }], timeout: 3 * 60 * 1000, expect: { timeout: 10 * 1000 }, - globalSetup: require.resolve("./globalSetup"), + globalSetup: require.resolve("./migration.serverRootPath.globalSetup"), }); diff --git a/ui/litellm-dashboard/e2e_tests/migration.serverRootPath.globalSetup.ts b/ui/litellm-dashboard/e2e_tests/migration.serverRootPath.globalSetup.ts new file mode 100644 index 00000000000..d11f49dae74 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/migration.serverRootPath.globalSetup.ts @@ -0,0 +1,12 @@ +import globalSetup from "./globalSetup"; + +export default async function migrationServerRootPathGlobalSetup() { + if (!process.env.SERVER_ROOT_PATH) { + throw new Error( + "migration.serverRootPath.config.ts requires SERVER_ROOT_PATH to be set (e.g. SERVER_ROOT_PATH=/litellm). " + + "Without it this config silently re-runs the default mount and never exercises the prefix. " + + "For the root-less run use the default playwright.config.ts (npm run e2e:migration).", + ); + } + await globalSetup(); +} diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 2afc145d15b..828edde278b 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -18,6 +18,7 @@ "@types/papaparse": "5.5.2", "antd": "5.29.3", "cva": "1.0.0-beta.4", + "date-fns": "3.6.0", "dayjs": "1.11.19", "jwt-decode": "4.0.0", "lucide-react": "0.513.0", @@ -31,7 +32,6 @@ "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", "react-syntax-highlighter": "15.6.6", - "remark-gfm": "4.0.1", "tailwind-merge": "3.4.0", "uuid": "14.0.0" }, @@ -8601,16 +8601,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/markdown-table": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz", - "integrity": "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -8620,34 +8610,6 @@ "node": ">= 0.4" } }, - "node_modules/mdast-util-find-and-replace": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.2.tgz", - "integrity": "sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "escape-string-regexp": "^5.0.0", - "unist-util-is": "^6.0.0", - "unist-util-visit-parents": "^6.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-find-and-replace/node_modules/escape-string-regexp": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", - "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/mdast-util-from-markdown": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.3.tgz", @@ -8672,107 +8634,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/mdast-util-gfm": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.1.0.tgz", - "integrity": "sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ==", - "license": "MIT", - "dependencies": { - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-gfm-autolink-literal": "^2.0.0", - "mdast-util-gfm-footnote": "^2.0.0", - "mdast-util-gfm-strikethrough": "^2.0.0", - "mdast-util-gfm-table": "^2.0.0", - "mdast-util-gfm-task-list-item": "^2.0.0", - "mdast-util-to-markdown": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-gfm-autolink-literal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-2.0.1.tgz", - "integrity": "sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "ccount": "^2.0.0", - "devlop": "^1.0.0", - "mdast-util-find-and-replace": "^3.0.0", - "micromark-util-character": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-gfm-footnote": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-2.1.0.tgz", - "integrity": "sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "devlop": "^1.1.0", - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-to-markdown": "^2.0.0", - "micromark-util-normalize-identifier": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-gfm-strikethrough": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-2.0.0.tgz", - "integrity": "sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-to-markdown": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-gfm-table": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-2.0.0.tgz", - "integrity": "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "devlop": "^1.0.0", - "markdown-table": "^3.0.0", - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-to-markdown": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-gfm-task-list-item": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-2.0.0.tgz", - "integrity": "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "devlop": "^1.0.0", - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-to-markdown": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, "node_modules/mdast-util-mdx-expression": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.1.tgz", @@ -8987,127 +8848,6 @@ "micromark-util-types": "^2.0.0" } }, - "node_modules/micromark-extension-gfm": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", - "integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==", - "license": "MIT", - "dependencies": { - "micromark-extension-gfm-autolink-literal": "^2.0.0", - "micromark-extension-gfm-footnote": "^2.0.0", - "micromark-extension-gfm-strikethrough": "^2.0.0", - "micromark-extension-gfm-table": "^2.0.0", - "micromark-extension-gfm-tagfilter": "^2.0.0", - "micromark-extension-gfm-task-list-item": "^2.0.0", - "micromark-util-combine-extensions": "^2.0.0", - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-gfm-autolink-literal": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", - "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", - "license": "MIT", - "dependencies": { - "micromark-util-character": "^2.0.0", - "micromark-util-sanitize-uri": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-gfm-footnote": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", - "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", - "license": "MIT", - "dependencies": { - "devlop": "^1.0.0", - "micromark-core-commonmark": "^2.0.0", - "micromark-factory-space": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-normalize-identifier": "^2.0.0", - "micromark-util-sanitize-uri": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-gfm-strikethrough": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz", - "integrity": "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==", - "license": "MIT", - "dependencies": { - "devlop": "^1.0.0", - "micromark-util-chunked": "^2.0.0", - "micromark-util-classify-character": "^2.0.0", - "micromark-util-resolve-all": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-gfm-table": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.1.tgz", - "integrity": "sha512-t2OU/dXXioARrC6yWfJ4hqB7rct14e8f7m0cbI5hUmDyyIlwv5vEtooptH8INkbLzOatzKuVbQmAYcbWoyz6Dg==", - "license": "MIT", - "dependencies": { - "devlop": "^1.0.0", - "micromark-factory-space": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-gfm-tagfilter": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz", - "integrity": "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==", - "license": "MIT", - "dependencies": { - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-gfm-task-list-item": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz", - "integrity": "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==", - "license": "MIT", - "dependencies": { - "devlop": "^1.0.0", - "micromark-factory-space": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, "node_modules/micromark-factory-destination": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", @@ -11701,24 +11441,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/remark-gfm": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.1.tgz", - "integrity": "sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "mdast-util-gfm": "^3.0.0", - "micromark-extension-gfm": "^3.0.0", - "remark-parse": "^11.0.0", - "remark-stringify": "^11.0.0", - "unified": "^11.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, "node_modules/remark-parse": { "version": "11.0.0", "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", @@ -11752,21 +11474,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/remark-stringify": { - "version": "11.0.0", - "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-11.0.0.tgz", - "integrity": "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "mdast-util-to-markdown": "^2.0.0", - "unified": "^11.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index a8948f4be34..0097e8c6559 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -34,6 +34,7 @@ "@types/papaparse": "5.5.2", "antd": "5.29.3", "cva": "1.0.0-beta.4", + "date-fns": "3.6.0", "dayjs": "1.11.19", "jwt-decode": "4.0.0", "lucide-react": "0.513.0", @@ -47,7 +48,6 @@ "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", "react-syntax-highlighter": "15.6.6", - "remark-gfm": "4.0.1", "tailwind-merge": "3.4.0", "uuid": "14.0.0" }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/components/index.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/components/index.tsx deleted file mode 100644 index 8b1c720148a..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/memory/components/index.tsx +++ /dev/null @@ -1 +0,0 @@ -export { MemoryView, default } from "./MemoryView"; From 3d644e1f9dd5985a7796051b9b08b696e7e19ba7 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 1 Jul 2026 20:14:07 -0700 Subject: [PATCH 07/18] refactor(ui): colocate users page into route-level _components (#31897) Moves the user-management component tree (view_users plus BulkEditUsers, edit_user, DefaultUserSettings, user_edit_view, and the view_users table/columns/info-view) out of the shared src/components dump into the users route segment under _components, now that the app router owns the route. The page imports from a trimmed ./_components barrel UserInfo moves into networking.tsx beside UserListResponse, its real owner: networking defines the user API response shapes that embed it, and previously reached up into a view folder (components/view_users/types) to import the type. Defining it in networking removes that backwards data-layer-to-view dependency and drains the view_users/ folder entirely. CreateUserButton and onboarding_link stay in components/ since the create-key flow also consumes them Relative imports in the moved files are rewritten to @/components/* absolute paths, and the eight pre-existing eslint-suppressions entries are re-keyed to the new paths so the move stays behavior and lint neutral Verified: the moved suites pass with the same 75 assertions as before the move, tsc and eslint are clean, and next build compiles the /users route --- ui/litellm-dashboard/eslint-suppressions.json | 16 ++++++++-------- .../users/_components}/BulkEditUsers.test.tsx | 8 ++++---- .../users/_components}/BulkEditUsers.tsx | 4 ++-- .../_components}/DefaultUserSettings.test.tsx | 8 ++++---- .../users/_components}/DefaultUserSettings.tsx | 10 ++++++---- .../users/_components}/edit_user.tsx | 4 ++-- .../(dashboard)/users/_components/index.tsx | 1 + .../users/_components}/user_edit_view.test.tsx | 6 +++--- .../users/_components}/user_edit_view.tsx | 8 ++++---- .../users/_components}/view_users.test.tsx | 4 ++-- .../users/_components}/view_users.tsx | 14 +++++++------- .../users/_components}/view_users/columns.tsx | 2 +- .../_components}/view_users/table.test.tsx | 2 +- .../users/_components}/view_users/table.tsx | 2 +- .../view_users/user_info_view.test.tsx | 2 +- .../_components}/view_users/user_info_view.tsx | 12 ++++++------ .../src/app/(dashboard)/users/page.tsx | 2 +- .../src/components/networking.tsx | 18 ++++++++++++++++-- .../src/components/view_users/types.ts | 15 --------------- .../tests/CreateKeyPage.expiredToken.test.tsx | 1 - 20 files changed, 70 insertions(+), 69 deletions(-) rename ui/litellm-dashboard/src/{components => app/(dashboard)/users/_components}/BulkEditUsers.test.tsx (97%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/users/_components}/BulkEditUsers.tsx (99%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/users/_components}/DefaultUserSettings.test.tsx (94%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/users/_components}/DefaultUserSettings.tsx (97%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/users/_components}/edit_user.tsx (95%) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/users/_components/index.tsx rename ui/litellm-dashboard/src/{components => app/(dashboard)/users/_components}/user_edit_view.test.tsx (98%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/users/_components}/user_edit_view.tsx (95%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/users/_components}/view_users.test.tsx (98%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/users/_components}/view_users.tsx (97%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/users/_components}/view_users/columns.tsx (99%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/users/_components}/view_users/table.test.tsx (99%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/users/_components}/view_users/table.tsx (99%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/users/_components}/view_users/user_info_view.test.tsx (99%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/users/_components}/view_users/user_info_view.tsx (98%) delete mode 100644 ui/litellm-dashboard/src/components/view_users/types.ts diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 7a3c8f4a42c..7ea13deb934 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -376,7 +376,7 @@ "count": 1 } }, - "src/components/DefaultUserSettings.tsx": { + "src/app/(dashboard)/users/_components/DefaultUserSettings.tsx": { "no-restricted-imports": { "count": 1 } @@ -1001,7 +1001,7 @@ "count": 1 } }, - "src/components/edit_user.tsx": { + "src/app/(dashboard)/users/_components/edit_user.tsx": { "no-restricted-imports": { "count": 1 } @@ -1957,12 +1957,12 @@ "count": 2 } }, - "src/components/user_edit_view.test.tsx": { + "src/app/(dashboard)/users/_components/user_edit_view.test.tsx": { "react/display-name": { "count": 1 } }, - "src/components/user_edit_view.tsx": { + "src/app/(dashboard)/users/_components/user_edit_view.tsx": { "no-restricted-imports": { "count": 1 }, @@ -2047,7 +2047,7 @@ "count": 2 } }, - "src/components/view_users.tsx": { + "src/app/(dashboard)/users/_components/view_users.tsx": { "no-restricted-imports": { "count": 1 }, @@ -2055,7 +2055,7 @@ "count": 1 } }, - "src/components/view_users/columns.tsx": { + "src/app/(dashboard)/users/_components/view_users/columns.tsx": { "max-params": { "count": 1 }, @@ -2063,12 +2063,12 @@ "count": 1 } }, - "src/components/view_users/table.tsx": { + "src/app/(dashboard)/users/_components/view_users/table.tsx": { "no-restricted-imports": { "count": 1 } }, - "src/components/view_users/user_info_view.tsx": { + "src/app/(dashboard)/users/_components/view_users/user_info_view.tsx": { "no-restricted-imports": { "count": 1 }, diff --git a/ui/litellm-dashboard/src/components/BulkEditUsers.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.test.tsx similarity index 97% rename from ui/litellm-dashboard/src/components/BulkEditUsers.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.test.tsx index f0af295017e..f16f5325952 100644 --- a/ui/litellm-dashboard/src/components/BulkEditUsers.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.test.tsx @@ -1,11 +1,11 @@ import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi, beforeEach } from "vitest"; -import { renderWithProviders, screen, waitFor } from "../../tests/test-utils"; +import { renderWithProviders, screen, waitFor } from "../../../../../tests/test-utils"; import BulkEditUserModal from "./BulkEditUsers"; -import { userBulkUpdateUserCall, teamBulkMemberAddCall } from "./networking"; -import NotificationsManager from "./molecules/notifications_manager"; +import { userBulkUpdateUserCall, teamBulkMemberAddCall } from "@/components/networking"; +import NotificationsManager from "@/components/molecules/notifications_manager"; -vi.mock("./networking", () => ({ +vi.mock("@/components/networking", () => ({ userBulkUpdateUserCall: vi.fn(), teamBulkMemberAddCall: vi.fn(), })); diff --git a/ui/litellm-dashboard/src/components/BulkEditUsers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx similarity index 99% rename from ui/litellm-dashboard/src/components/BulkEditUsers.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx index 7bcfd793604..f22de171d68 100644 --- a/ui/litellm-dashboard/src/components/BulkEditUsers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx @@ -1,8 +1,8 @@ import React, { useState } from "react"; import { Modal, Typography, Divider, Table, Select, InputNumber, Card, Space, Checkbox } from "antd"; -import { userBulkUpdateUserCall, teamBulkMemberAddCall, Member } from "./networking"; +import { userBulkUpdateUserCall, teamBulkMemberAddCall, Member } from "@/components/networking"; import { UserEditView } from "./user_edit_view"; -import NotificationsManager from "./molecules/notifications_manager"; +import NotificationsManager from "@/components/molecules/notifications_manager"; import MessageManager from "@/components/molecules/message_manager"; const { Text, Title } = Typography; diff --git a/ui/litellm-dashboard/src/components/DefaultUserSettings.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/DefaultUserSettings.test.tsx similarity index 94% rename from ui/litellm-dashboard/src/components/DefaultUserSettings.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/users/_components/DefaultUserSettings.test.tsx index 78d50fa7f31..06dafcfcffd 100644 --- a/ui/litellm-dashboard/src/components/DefaultUserSettings.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/DefaultUserSettings.test.tsx @@ -1,15 +1,15 @@ import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; import DefaultUserSettings from "./DefaultUserSettings"; -import * as networking from "./networking"; +import * as networking from "@/components/networking"; -vi.mock("./networking", () => ({ +vi.mock("@/components/networking", () => ({ getInternalUserSettings: vi.fn(), updateInternalUserSettings: vi.fn(), modelAvailableCall: vi.fn(), })); -vi.mock("./common_components/budget_duration_dropdown", () => ({ +vi.mock("@/components/common_components/budget_duration_dropdown", () => ({ default: ({ value, onChange }: { value: string | null; onChange: (value: string | null) => void }) => (