From 5215c2d3f77cb7cad6db861f323a4e892f5c16bf Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 28 Oct 2026 14:59:23 -0700 Subject: [PATCH 01/27] fix(pricing): add databricks priority pay per token prices (#48891) * fix(pricing): add databricks priority pay per token prices * fix(pricing): bill gemini 3.6 flash cache writes at its databricks input rate * fix(pricing): drop image token priority rate the cost map schema does not carry --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 233 +++++++++++++++--- model_prices_and_context_window.json | 233 +++++++++++++++--- .../test_databricks_cost_calculator.py | 26 ++ 3 files changed, 428 insertions(+), 64 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3e0e86012a4..1f2bd7d7837 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -21589,7 +21589,13 @@ ], "supports_vision": true, "supports_audio_input": true, - "supports_video_input": true + "supports_video_input": true, + "input_cost_per_token_priority": 5.6252e-07, + "output_cost_per_token_priority": 3.37498e-06, + "cache_read_input_token_cost_priority": 5.628e-08, + "cache_creation_input_token_cost_priority": 5.6252e-07, + "input_cost_per_audio_token_priority": 1.12497e-06, + "cache_read_input_audio_token_cost_priority": 1.1249e-07 }, "databricks/databricks-gemini-3-1-flash-image": { "cache_creation_input_token_cost": 6.2503e-07, @@ -21676,7 +21682,14 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 4.50002e-06, + "output_cost_per_token_priority": 2.699998e-05, + "cache_read_input_token_cost_priority": 4.5003e-07, + "cache_creation_input_token_cost_priority": 4.50002e-06, + "input_cost_per_token_above_200k_tokens_priority": 8.99997e-06, + "output_cost_per_token_above_200k_tokens_priority": 4.049997e-05, + "cache_read_input_token_cost_above_200k_tokens_priority": 8.9999e-07 }, "databricks/databricks-gemini-3-flash": { "cache_creation_input_token_cost": 6.2503e-07, @@ -21710,7 +21723,13 @@ ], "supports_vision": true, "supports_audio_input": true, - "supports_video_input": true + "supports_video_input": true, + "input_cost_per_token_priority": 1.12497e-06, + "output_cost_per_token_priority": 6.75003e-06, + "cache_read_input_token_cost_priority": 1.1249e-07, + "cache_creation_input_token_cost_priority": 1.12497e-06, + "input_cost_per_audio_token_priority": 2.25001e-06, + "cache_read_input_audio_token_cost_priority": 2.2498e-07 }, "databricks/databricks-gemini-3-pro": { "cache_creation_input_token_cost": 2.49998e-06, @@ -21757,7 +21776,11 @@ "supports_reasoning": true, "supports_sampling_params": false, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.35002e-06, + "output_cost_per_token_priority": 6.75003e-06, + "cache_read_input_token_cost_priority": 1.3503e-07, + "cache_creation_input_token_cost_priority": 1.35002e-06 }, "databricks/databricks-gemini-3-7-flash": { "cache_creation_input_token_cost": 7.4998e-07, @@ -21785,10 +21808,14 @@ "supports_reasoning": true, "supports_sampling_params": false, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.35002e-06, + "output_cost_per_token_priority": 6.75003e-06, + "cache_read_input_token_cost_priority": 1.3503e-07, + "cache_creation_input_token_cost_priority": 1.35002e-06 }, "databricks/databricks-gemini-3-6-flash": { - "cache_creation_input_token_cost": 1.87502e-06, + "cache_creation_input_token_cost": 7.4998e-07, "cache_read_input_token_cost": 7.497e-08, "input_cost_per_token": 7.4998e-07, "input_dbu_cost_per_token": 1.0714e-05, @@ -21816,7 +21843,11 @@ "supports_reasoning": true, "supports_sampling_params": false, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.35002e-06, + "output_cost_per_token_priority": 6.75003e-06, + "cache_read_input_token_cost_priority": 1.3503e-07, + "cache_creation_input_token_cost_priority": 1.35002e-06 }, "databricks/databricks-gemini-3-5-flash": { "cache_creation_input_token_cost": 1.87502e-06, @@ -21846,7 +21877,11 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 3.37498e-06, + "output_cost_per_token_priority": 2.025002e-05, + "cache_read_input_token_cost_priority": 3.3747e-07, + "cache_creation_input_token_cost_priority": 3.37498e-06 }, "databricks/databricks-gemini-3-5-flash-lite": { "cache_creation_input_token_cost": 3.7499e-07, @@ -21878,7 +21913,11 @@ "supports_response_schema": true, "supports_sampling_params": false, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 6.7501e-07, + "output_cost_per_token_priority": 5.62499e-06, + "cache_read_input_token_cost_priority": 6.748e-08, + "cache_creation_input_token_cost_priority": 6.7501e-07 }, "databricks/databricks-gemma-3-12b": { "cache_creation_input_token_cost": 1.5001e-07, @@ -21921,7 +21960,11 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "input_cost_per_token_priority": 2.45e-06, + "output_cost_per_token_priority": 7.7e-06, + "cache_read_input_token_cost_priority": 4.55e-07, + "cache_creation_input_token_cost_priority": 2.45e-06 }, "databricks/databricks-glm-5-3": { "cache_creation_input_token_cost": 1.4e-06, @@ -21951,7 +21994,11 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": false, - "thinking_always_on": true + "thinking_always_on": true, + "input_cost_per_token_priority": 2.45e-06, + "output_cost_per_token_priority": 7.7e-06, + "cache_read_input_token_cost_priority": 4.55e-07, + "cache_creation_input_token_cost_priority": 2.45e-06 }, "databricks/databricks-glm-5-3-flash": { "cache_creation_input_token_cost": 1.5001e-07, @@ -21980,7 +22027,11 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "thinking_always_on": true + "thinking_always_on": true, + "input_cost_per_token_priority": 2.625e-07, + "output_cost_per_token_priority": 8.75e-07, + "cache_read_input_token_cost_priority": 5.25e-08, + "cache_creation_input_token_cost_priority": 2.625e-07 }, "databricks/databricks-gpt-5": { "cache_creation_input_token_cost": 1.24999e-06, @@ -21999,7 +22050,11 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, "supports_prompt_caching": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "input_cost_per_token_priority": 2.49998e-06, + "output_cost_per_token_priority": 1.999998e-05, + "cache_read_input_token_cost_priority": 2.4997e-07, + "cache_creation_input_token_cost_priority": 2.49998e-06 }, "databricks/databricks-gpt-5-1": { "cache_creation_input_token_cost": 1.24999e-06, @@ -22018,7 +22073,11 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, "supports_prompt_caching": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "input_cost_per_token_priority": 2.49998e-06, + "output_cost_per_token_priority": 1.999998e-05, + "cache_read_input_token_cost_priority": 2.4997e-07, + "cache_creation_input_token_cost_priority": 2.49998e-06 }, "databricks/databricks-gpt-5-2": { "cache_creation_input_token_cost": 1.75e-06, @@ -22037,7 +22096,11 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, "supports_prompt_caching": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "input_cost_per_token_priority": 3.5e-06, + "output_cost_per_token_priority": 2.8e-05, + "cache_read_input_token_cost_priority": 3.5e-07, + "cache_creation_input_token_cost_priority": 3.5e-06 }, "databricks/databricks-gpt-5-3-codex": { "cache_creation_input_token_cost": 1.75e-06, @@ -22056,7 +22119,11 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, "supports_prompt_caching": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "input_cost_per_token_priority": 3.5e-06, + "output_cost_per_token_priority": 2.8e-05, + "cache_read_input_token_cost_priority": 3.5e-07, + "cache_creation_input_token_cost_priority": 3.5e-06 }, "databricks/databricks-gpt-5-4": { "cache_creation_input_token_cost": 2.49998e-06, @@ -22087,7 +22154,14 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 5.00003e-06, + "output_cost_per_token_priority": 2.999997e-05, + "cache_read_input_token_cost_priority": 5.0001e-07, + "cache_creation_input_token_cost_priority": 5.00003e-06, + "input_cost_per_token_above_272k_tokens_priority": 5.00003e-06, + "output_cost_per_token_above_272k_tokens_priority": 2.999997e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 5.0001e-07 }, "databricks/databricks-gpt-5-4-mini": { "cache_creation_input_token_cost": 7.4998e-07, @@ -22115,7 +22189,11 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.50003e-06, + "output_cost_per_token_priority": 8.99997e-06, + "cache_read_input_token_cost_priority": 1.5001e-07, + "cache_creation_input_token_cost_priority": 1.50003e-06 }, "databricks/databricks-gpt-5-4-nano": { "cache_creation_input_token_cost": 1.9999e-07, @@ -22175,7 +22253,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 8.00002e-06, + "output_cost_per_token_priority": 4.000003e-05, + "cache_read_input_token_cost_priority": 8.0003e-07, + "cache_creation_input_token_cost_priority": 9.99999e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.599997e-05, + "output_cost_per_token_above_272k_tokens_priority": 6.000001e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.59999e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1.999998e-05 }, "databricks/databricks-gpt-5-6-terra": { "cache_creation_input_token_cost": 2.49998e-06, @@ -22207,7 +22293,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 4.00001e-06, + "output_cost_per_token_priority": 2.399999e-05, + "cache_read_input_token_cost_priority": 3.9998e-07, + "cache_creation_input_token_cost_priority": 5.00003e-06, + "input_cost_per_token_above_272k_tokens_priority": 8.00002e-06, + "output_cost_per_token_above_272k_tokens_priority": 3.600002e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 8.0003e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06 }, "databricks/databricks-gpt-5-6-luna": { "cache_creation_input_token_cost": 2.4997e-07, @@ -22239,7 +22333,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 3.9998e-07, + "output_cost_per_token_priority": 2.40002e-06, + "cache_read_input_token_cost_priority": 3.997e-08, + "cache_creation_input_token_cost_priority": 5.0001e-07, + "input_cost_per_token_above_272k_tokens_priority": 8.0003e-07, + "output_cost_per_token_above_272k_tokens_priority": 3.60003e-06, + "cache_read_input_token_cost_above_272k_tokens_priority": 8.001e-08, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1.00002e-06 }, "databricks/databricks-gpt-5-5": { "cache_creation_input_token_cost": 5.00003e-06, @@ -22270,7 +22372,14 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.249997e-05, + "output_cost_per_token_priority": 7.50001e-05, + "cache_read_input_token_cost_priority": 1.24999e-06, + "cache_creation_input_token_cost_priority": 1.249997e-05, + "input_cost_per_token_above_272k_tokens_priority": 1.249997e-05, + "output_cost_per_token_above_272k_tokens_priority": 7.50001e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.24999e-06 }, "databricks/databricks-gpt-5-5-pro": { "cache_creation_input_token_cost": 2.999997e-05, @@ -22319,7 +22428,11 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, "supports_prompt_caching": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "input_cost_per_token_priority": 4.5003e-07, + "output_cost_per_token_priority": 3.60003e-06, + "cache_read_input_token_cost_priority": 4.501e-08, + "cache_creation_input_token_cost_priority": 4.5003e-07 }, "databricks/databricks-gpt-5-nano": { "cache_creation_input_token_cost": 4.998e-08, @@ -22469,7 +22582,11 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 5.25e-06, + "output_cost_per_token_priority": 2.625e-05, + "cache_read_input_token_cost_priority": 5.25e-07, + "cache_creation_input_token_cost_priority": 5.25e-06 }, "databricks/system.ai.kimi-k3": { "litellm_provider": "databricks", @@ -22551,7 +22668,11 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": false, - "thinking_always_on": true + "thinking_always_on": true, + "input_cost_per_token_priority": 3.85e-07, + "output_cost_per_token_priority": 3.85e-06, + "cache_read_input_token_cost_priority": 3.85e-07, + "cache_creation_input_token_cost_priority": 3.85e-07 }, "databricks/databricks-qwen3-next-80b-a3b-instruct": { "cache_creation_input_token_cost": 1.5001e-07, @@ -77774,7 +77895,11 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 5.25e-07, + "output_cost_per_token_priority": 2.1e-06, + "cache_read_input_token_cost_priority": 5.25e-08, + "cache_creation_input_token_cost_priority": 5.25e-07 }, "databricks/databricks-deepseek-v4-1-flash-1017": { "cache_creation_input_token_cost": 3.0002e-07, @@ -77897,7 +78022,11 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 4.50002e-06, + "output_cost_per_token_priority": 2.250003e-05, + "cache_read_input_token_cost_priority": 2.2498e-07, + "cache_creation_input_token_cost_priority": 4.50002e-06 }, "databricks/databricks-gpt-5-4-pro": { "cache_creation_input_token_cost": 2.999997e-05, @@ -77959,7 +78088,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 4.00001e-06, + "output_cost_per_token_priority": 1.99997e-05, + "cache_read_input_token_cost_priority": 1.9999e-07, + "cache_creation_input_token_cost_priority": 5.00003e-06, + "input_cost_per_token_above_272k_tokens_priority": 8.00002e-06, + "output_cost_per_token_above_272k_tokens_priority": 2.99999e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 3.9998e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06 }, "databricks/databricks-gpt-6-1-terra": { "cache_creation_input_token_cost": 1.24999e-06, @@ -77991,7 +78128,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.99997e-06, + "output_cost_per_token_priority": 9.99999e-06, + "cache_read_input_token_cost_priority": 1.0003e-07, + "cache_creation_input_token_cost_priority": 2.49998e-06, + "input_cost_per_token_above_272k_tokens_priority": 4.00001e-06, + "output_cost_per_token_above_272k_tokens_priority": 1.500002e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.9999e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5.00003e-06 }, "databricks/databricks-gpt-6-astra": { "cache_creation_input_token_cost": 1.249997e-05, @@ -78023,7 +78168,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.999998e-05, + "output_cost_per_token_priority": 9.99999e-05, + "cache_read_input_token_cost_priority": 1.99997e-06, + "cache_creation_input_token_cost_priority": 2.500001e-05, + "input_cost_per_token_above_272k_tokens_priority": 4.000003e-05, + "output_cost_per_token_above_272k_tokens_priority": 0.0001500002, + "cache_read_input_token_cost_above_272k_tokens_priority": 4.00001e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5.000002e-05 }, "databricks/databricks-gpt-6-luna": { "cache_creation_input_token_cost": 1.2502e-07, @@ -78055,7 +78208,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.9999e-07, + "output_cost_per_token_priority": 1.0003e-06, + "cache_read_input_token_cost_priority": 2.002e-08, + "cache_creation_input_token_cost_priority": 2.4997e-07, + "input_cost_per_token_above_272k_tokens_priority": 3.9998e-07, + "output_cost_per_token_above_272k_tokens_priority": 1.5001e-06, + "cache_read_input_token_cost_above_272k_tokens_priority": 3.997e-08, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5.0001e-07 }, "databricks/databricks-gpt-6-sol": { "cache_creation_input_token_cost": 2.49998e-06, @@ -78087,7 +78248,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 4.00001e-06, + "output_cost_per_token_priority": 1.99997e-05, + "cache_read_input_token_cost_priority": 3.9998e-07, + "cache_creation_input_token_cost_priority": 5.00003e-06, + "input_cost_per_token_above_272k_tokens_priority": 8.00002e-06, + "output_cost_per_token_above_272k_tokens_priority": 2.99999e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 8.0003e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06 }, "databricks/databricks-gpt-image-1": { "cache_creation_input_token_cost": 5.00003e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3e0e86012a4..1f2bd7d7837 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -21589,7 +21589,13 @@ ], "supports_vision": true, "supports_audio_input": true, - "supports_video_input": true + "supports_video_input": true, + "input_cost_per_token_priority": 5.6252e-07, + "output_cost_per_token_priority": 3.37498e-06, + "cache_read_input_token_cost_priority": 5.628e-08, + "cache_creation_input_token_cost_priority": 5.6252e-07, + "input_cost_per_audio_token_priority": 1.12497e-06, + "cache_read_input_audio_token_cost_priority": 1.1249e-07 }, "databricks/databricks-gemini-3-1-flash-image": { "cache_creation_input_token_cost": 6.2503e-07, @@ -21676,7 +21682,14 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 4.50002e-06, + "output_cost_per_token_priority": 2.699998e-05, + "cache_read_input_token_cost_priority": 4.5003e-07, + "cache_creation_input_token_cost_priority": 4.50002e-06, + "input_cost_per_token_above_200k_tokens_priority": 8.99997e-06, + "output_cost_per_token_above_200k_tokens_priority": 4.049997e-05, + "cache_read_input_token_cost_above_200k_tokens_priority": 8.9999e-07 }, "databricks/databricks-gemini-3-flash": { "cache_creation_input_token_cost": 6.2503e-07, @@ -21710,7 +21723,13 @@ ], "supports_vision": true, "supports_audio_input": true, - "supports_video_input": true + "supports_video_input": true, + "input_cost_per_token_priority": 1.12497e-06, + "output_cost_per_token_priority": 6.75003e-06, + "cache_read_input_token_cost_priority": 1.1249e-07, + "cache_creation_input_token_cost_priority": 1.12497e-06, + "input_cost_per_audio_token_priority": 2.25001e-06, + "cache_read_input_audio_token_cost_priority": 2.2498e-07 }, "databricks/databricks-gemini-3-pro": { "cache_creation_input_token_cost": 2.49998e-06, @@ -21757,7 +21776,11 @@ "supports_reasoning": true, "supports_sampling_params": false, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.35002e-06, + "output_cost_per_token_priority": 6.75003e-06, + "cache_read_input_token_cost_priority": 1.3503e-07, + "cache_creation_input_token_cost_priority": 1.35002e-06 }, "databricks/databricks-gemini-3-7-flash": { "cache_creation_input_token_cost": 7.4998e-07, @@ -21785,10 +21808,14 @@ "supports_reasoning": true, "supports_sampling_params": false, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.35002e-06, + "output_cost_per_token_priority": 6.75003e-06, + "cache_read_input_token_cost_priority": 1.3503e-07, + "cache_creation_input_token_cost_priority": 1.35002e-06 }, "databricks/databricks-gemini-3-6-flash": { - "cache_creation_input_token_cost": 1.87502e-06, + "cache_creation_input_token_cost": 7.4998e-07, "cache_read_input_token_cost": 7.497e-08, "input_cost_per_token": 7.4998e-07, "input_dbu_cost_per_token": 1.0714e-05, @@ -21816,7 +21843,11 @@ "supports_reasoning": true, "supports_sampling_params": false, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.35002e-06, + "output_cost_per_token_priority": 6.75003e-06, + "cache_read_input_token_cost_priority": 1.3503e-07, + "cache_creation_input_token_cost_priority": 1.35002e-06 }, "databricks/databricks-gemini-3-5-flash": { "cache_creation_input_token_cost": 1.87502e-06, @@ -21846,7 +21877,11 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 3.37498e-06, + "output_cost_per_token_priority": 2.025002e-05, + "cache_read_input_token_cost_priority": 3.3747e-07, + "cache_creation_input_token_cost_priority": 3.37498e-06 }, "databricks/databricks-gemini-3-5-flash-lite": { "cache_creation_input_token_cost": 3.7499e-07, @@ -21878,7 +21913,11 @@ "supports_response_schema": true, "supports_sampling_params": false, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 6.7501e-07, + "output_cost_per_token_priority": 5.62499e-06, + "cache_read_input_token_cost_priority": 6.748e-08, + "cache_creation_input_token_cost_priority": 6.7501e-07 }, "databricks/databricks-gemma-3-12b": { "cache_creation_input_token_cost": 1.5001e-07, @@ -21921,7 +21960,11 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "input_cost_per_token_priority": 2.45e-06, + "output_cost_per_token_priority": 7.7e-06, + "cache_read_input_token_cost_priority": 4.55e-07, + "cache_creation_input_token_cost_priority": 2.45e-06 }, "databricks/databricks-glm-5-3": { "cache_creation_input_token_cost": 1.4e-06, @@ -21951,7 +21994,11 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": false, - "thinking_always_on": true + "thinking_always_on": true, + "input_cost_per_token_priority": 2.45e-06, + "output_cost_per_token_priority": 7.7e-06, + "cache_read_input_token_cost_priority": 4.55e-07, + "cache_creation_input_token_cost_priority": 2.45e-06 }, "databricks/databricks-glm-5-3-flash": { "cache_creation_input_token_cost": 1.5001e-07, @@ -21980,7 +22027,11 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "thinking_always_on": true + "thinking_always_on": true, + "input_cost_per_token_priority": 2.625e-07, + "output_cost_per_token_priority": 8.75e-07, + "cache_read_input_token_cost_priority": 5.25e-08, + "cache_creation_input_token_cost_priority": 2.625e-07 }, "databricks/databricks-gpt-5": { "cache_creation_input_token_cost": 1.24999e-06, @@ -21999,7 +22050,11 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, "supports_prompt_caching": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "input_cost_per_token_priority": 2.49998e-06, + "output_cost_per_token_priority": 1.999998e-05, + "cache_read_input_token_cost_priority": 2.4997e-07, + "cache_creation_input_token_cost_priority": 2.49998e-06 }, "databricks/databricks-gpt-5-1": { "cache_creation_input_token_cost": 1.24999e-06, @@ -22018,7 +22073,11 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, "supports_prompt_caching": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "input_cost_per_token_priority": 2.49998e-06, + "output_cost_per_token_priority": 1.999998e-05, + "cache_read_input_token_cost_priority": 2.4997e-07, + "cache_creation_input_token_cost_priority": 2.49998e-06 }, "databricks/databricks-gpt-5-2": { "cache_creation_input_token_cost": 1.75e-06, @@ -22037,7 +22096,11 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, "supports_prompt_caching": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "input_cost_per_token_priority": 3.5e-06, + "output_cost_per_token_priority": 2.8e-05, + "cache_read_input_token_cost_priority": 3.5e-07, + "cache_creation_input_token_cost_priority": 3.5e-06 }, "databricks/databricks-gpt-5-3-codex": { "cache_creation_input_token_cost": 1.75e-06, @@ -22056,7 +22119,11 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, "supports_prompt_caching": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "input_cost_per_token_priority": 3.5e-06, + "output_cost_per_token_priority": 2.8e-05, + "cache_read_input_token_cost_priority": 3.5e-07, + "cache_creation_input_token_cost_priority": 3.5e-06 }, "databricks/databricks-gpt-5-4": { "cache_creation_input_token_cost": 2.49998e-06, @@ -22087,7 +22154,14 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 5.00003e-06, + "output_cost_per_token_priority": 2.999997e-05, + "cache_read_input_token_cost_priority": 5.0001e-07, + "cache_creation_input_token_cost_priority": 5.00003e-06, + "input_cost_per_token_above_272k_tokens_priority": 5.00003e-06, + "output_cost_per_token_above_272k_tokens_priority": 2.999997e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 5.0001e-07 }, "databricks/databricks-gpt-5-4-mini": { "cache_creation_input_token_cost": 7.4998e-07, @@ -22115,7 +22189,11 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.50003e-06, + "output_cost_per_token_priority": 8.99997e-06, + "cache_read_input_token_cost_priority": 1.5001e-07, + "cache_creation_input_token_cost_priority": 1.50003e-06 }, "databricks/databricks-gpt-5-4-nano": { "cache_creation_input_token_cost": 1.9999e-07, @@ -22175,7 +22253,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 8.00002e-06, + "output_cost_per_token_priority": 4.000003e-05, + "cache_read_input_token_cost_priority": 8.0003e-07, + "cache_creation_input_token_cost_priority": 9.99999e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.599997e-05, + "output_cost_per_token_above_272k_tokens_priority": 6.000001e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.59999e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1.999998e-05 }, "databricks/databricks-gpt-5-6-terra": { "cache_creation_input_token_cost": 2.49998e-06, @@ -22207,7 +22293,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 4.00001e-06, + "output_cost_per_token_priority": 2.399999e-05, + "cache_read_input_token_cost_priority": 3.9998e-07, + "cache_creation_input_token_cost_priority": 5.00003e-06, + "input_cost_per_token_above_272k_tokens_priority": 8.00002e-06, + "output_cost_per_token_above_272k_tokens_priority": 3.600002e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 8.0003e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06 }, "databricks/databricks-gpt-5-6-luna": { "cache_creation_input_token_cost": 2.4997e-07, @@ -22239,7 +22333,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 3.9998e-07, + "output_cost_per_token_priority": 2.40002e-06, + "cache_read_input_token_cost_priority": 3.997e-08, + "cache_creation_input_token_cost_priority": 5.0001e-07, + "input_cost_per_token_above_272k_tokens_priority": 8.0003e-07, + "output_cost_per_token_above_272k_tokens_priority": 3.60003e-06, + "cache_read_input_token_cost_above_272k_tokens_priority": 8.001e-08, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1.00002e-06 }, "databricks/databricks-gpt-5-5": { "cache_creation_input_token_cost": 5.00003e-06, @@ -22270,7 +22372,14 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.249997e-05, + "output_cost_per_token_priority": 7.50001e-05, + "cache_read_input_token_cost_priority": 1.24999e-06, + "cache_creation_input_token_cost_priority": 1.249997e-05, + "input_cost_per_token_above_272k_tokens_priority": 1.249997e-05, + "output_cost_per_token_above_272k_tokens_priority": 7.50001e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.24999e-06 }, "databricks/databricks-gpt-5-5-pro": { "cache_creation_input_token_cost": 2.999997e-05, @@ -22319,7 +22428,11 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, "supports_prompt_caching": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "input_cost_per_token_priority": 4.5003e-07, + "output_cost_per_token_priority": 3.60003e-06, + "cache_read_input_token_cost_priority": 4.501e-08, + "cache_creation_input_token_cost_priority": 4.5003e-07 }, "databricks/databricks-gpt-5-nano": { "cache_creation_input_token_cost": 4.998e-08, @@ -22469,7 +22582,11 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 5.25e-06, + "output_cost_per_token_priority": 2.625e-05, + "cache_read_input_token_cost_priority": 5.25e-07, + "cache_creation_input_token_cost_priority": 5.25e-06 }, "databricks/system.ai.kimi-k3": { "litellm_provider": "databricks", @@ -22551,7 +22668,11 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": false, - "thinking_always_on": true + "thinking_always_on": true, + "input_cost_per_token_priority": 3.85e-07, + "output_cost_per_token_priority": 3.85e-06, + "cache_read_input_token_cost_priority": 3.85e-07, + "cache_creation_input_token_cost_priority": 3.85e-07 }, "databricks/databricks-qwen3-next-80b-a3b-instruct": { "cache_creation_input_token_cost": 1.5001e-07, @@ -77774,7 +77895,11 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 5.25e-07, + "output_cost_per_token_priority": 2.1e-06, + "cache_read_input_token_cost_priority": 5.25e-08, + "cache_creation_input_token_cost_priority": 5.25e-07 }, "databricks/databricks-deepseek-v4-1-flash-1017": { "cache_creation_input_token_cost": 3.0002e-07, @@ -77897,7 +78022,11 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 4.50002e-06, + "output_cost_per_token_priority": 2.250003e-05, + "cache_read_input_token_cost_priority": 2.2498e-07, + "cache_creation_input_token_cost_priority": 4.50002e-06 }, "databricks/databricks-gpt-5-4-pro": { "cache_creation_input_token_cost": 2.999997e-05, @@ -77959,7 +78088,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 4.00001e-06, + "output_cost_per_token_priority": 1.99997e-05, + "cache_read_input_token_cost_priority": 1.9999e-07, + "cache_creation_input_token_cost_priority": 5.00003e-06, + "input_cost_per_token_above_272k_tokens_priority": 8.00002e-06, + "output_cost_per_token_above_272k_tokens_priority": 2.99999e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 3.9998e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06 }, "databricks/databricks-gpt-6-1-terra": { "cache_creation_input_token_cost": 1.24999e-06, @@ -77991,7 +78128,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.99997e-06, + "output_cost_per_token_priority": 9.99999e-06, + "cache_read_input_token_cost_priority": 1.0003e-07, + "cache_creation_input_token_cost_priority": 2.49998e-06, + "input_cost_per_token_above_272k_tokens_priority": 4.00001e-06, + "output_cost_per_token_above_272k_tokens_priority": 1.500002e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.9999e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5.00003e-06 }, "databricks/databricks-gpt-6-astra": { "cache_creation_input_token_cost": 1.249997e-05, @@ -78023,7 +78168,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.999998e-05, + "output_cost_per_token_priority": 9.99999e-05, + "cache_read_input_token_cost_priority": 1.99997e-06, + "cache_creation_input_token_cost_priority": 2.500001e-05, + "input_cost_per_token_above_272k_tokens_priority": 4.000003e-05, + "output_cost_per_token_above_272k_tokens_priority": 0.0001500002, + "cache_read_input_token_cost_above_272k_tokens_priority": 4.00001e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5.000002e-05 }, "databricks/databricks-gpt-6-luna": { "cache_creation_input_token_cost": 1.2502e-07, @@ -78055,7 +78208,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 1.9999e-07, + "output_cost_per_token_priority": 1.0003e-06, + "cache_read_input_token_cost_priority": 2.002e-08, + "cache_creation_input_token_cost_priority": 2.4997e-07, + "input_cost_per_token_above_272k_tokens_priority": 3.9998e-07, + "output_cost_per_token_above_272k_tokens_priority": 1.5001e-06, + "cache_read_input_token_cost_above_272k_tokens_priority": 3.997e-08, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5.0001e-07 }, "databricks/databricks-gpt-6-sol": { "cache_creation_input_token_cost": 2.49998e-06, @@ -78087,7 +78248,15 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "input_cost_per_token_priority": 4.00001e-06, + "output_cost_per_token_priority": 1.99997e-05, + "cache_read_input_token_cost_priority": 3.9998e-07, + "cache_creation_input_token_cost_priority": 5.00003e-06, + "input_cost_per_token_above_272k_tokens_priority": 8.00002e-06, + "output_cost_per_token_above_272k_tokens_priority": 2.99999e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 8.0003e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06 }, "databricks/databricks-gpt-image-1": { "cache_creation_input_token_cost": 5.00003e-06, diff --git a/tests/unit/llms/databricks/test_databricks_cost_calculator.py b/tests/unit/llms/databricks/test_databricks_cost_calculator.py index 34d3806d6eb..c35aa6e4cf0 100644 --- a/tests/unit/llms/databricks/test_databricks_cost_calculator.py +++ b/tests/unit/llms/databricks/test_databricks_cost_calculator.py @@ -253,3 +253,29 @@ def test_cost_per_token_bills_the_served_priority_tier( prompt_cost, completion_cost = cost_per_token(model="databricks/dbrx-tiered-test", usage=usage) assert prompt_cost == pytest.approx(30 * 0.001) assert completion_cost == pytest.approx(40 * 0.002) + + +RATES_WITHOUT_A_PRIORITY_FIELD: Final = frozenset({"input_cost_per_image_token"}) + + +def _is_standard_rate_key(key: str) -> bool: + return ( + ("_cost_per_" in key or key.endswith("_token_cost")) + and "dbu" not in key + and key not in RATES_WITHOUT_A_PRIORITY_FIELD + and not key.endswith(("_priority", "_flex", "_batches")) + ) + + +def test_every_databricks_priority_row_prices_every_standard_rate_at_the_priority_tier() -> None: + cost_map: Final = json.loads(MAIN_PRICES.read_text()) + priority_rows: Final = { + key: row + for key, row in cost_map.items() + if key.startswith("databricks/") and "input_cost_per_token_priority" in row + } + + assert priority_rows + for key, row in priority_rows.items(): + missing = sorted(f for f in row if _is_standard_rate_key(f) and f"{f}_priority" not in row) + assert not missing, f"{key} bills these at the standard rate on priority requests: {missing}" From b8fb50f70e82f5eed8a8d56cdb5b6b1ca4e1e7b7 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 28 Oct 2026 15:03:40 -0700 Subject: [PATCH 02/27] fix(ci): keep the spend logs guard approval through a later COMMENTED review (LIT-10242) (#48901) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ci_cd/spend_logs_guard.py | 11 ++++++++--- tests/unit/test_spend_logs_guard.py | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/ci_cd/spend_logs_guard.py b/ci_cd/spend_logs_guard.py index 64e33c4fb70..fac9d4ba5d2 100644 --- a/ci_cd/spend_logs_guard.py +++ b/ci_cd/spend_logs_guard.py @@ -27,7 +27,8 @@ WATCHED: Final = ( "litellm-proxy-extras/litellm_proxy_extras/schema.prisma", ) MIGRATIONS_PREFIX: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations/" -TABLE: Final = re.compile(r"(? str: def approved_by(reviews: Sequence[Review], login: str) -> bool: - """True when `login`'s most recent review on the pull request is an approval.""" - states: Final = tuple(review.state for review in reviews if review.user.login == login) + """True when `login`'s most recent review that changes GitHub's approval state is an approval. + + COMMENTED and PENDING reviews leave an earlier approval in force on GitHub, so they are skipped here too. + """ + own: Final = tuple(review.state for review in reviews if review.user.login == login) + states: Final = tuple(state for state in own if state in DECIDING_STATES) return bool(states) and states[-1] == "APPROVED" diff --git a/tests/unit/test_spend_logs_guard.py b/tests/unit/test_spend_logs_guard.py index 6a3355e1c52..9ac2c923632 100644 --- a/tests/unit/test_spend_logs_guard.py +++ b/tests/unit/test_spend_logs_guard.py @@ -3,6 +3,8 @@ import sys from pathlib import Path from typing import Final +import pytest + _SCRIPT: Final = Path(__file__).resolve().parents[2] / "ci_cd" / "spend_logs_guard.py" _SPEC: Final = importlib.util.spec_from_file_location("spend_logs_guard", _SCRIPT) assert _SPEC is not None and _SPEC.loader is not None @@ -35,6 +37,13 @@ class TestSpendLogHits: elsewhere: Final = _diff("litellm/proxy/utils.py", 'table = "LiteLLM_SpendLogs"', 5) assert guard.spend_log_hits(removed + other + elsewhere) == () + def test_a_table_whose_name_only_starts_with_the_watched_one_is_not_a_hit(self): + archive: Final = _diff("schema.prisma", "model LiteLLM_SpendLogsArchive {", 900) + index: Final = _diff("schema.prisma", " @@index([team_id], map: \"LiteLLM_SpendLogs_team_id_idx\")", 760) + assert guard.spend_log_hits(archive + index) == ( + 'schema.prisma: @@index([team_id], map: "LiteLLM_SpendLogs_team_id_idx")', + ) + def test_two_files_in_one_diff_are_attributed_to_their_own_path(self): diff: Final = _diff("litellm/proxy/schema.prisma", " a LiteLLM_SpendLogs", 1) + _diff( "schema.prisma", " b String", 1 @@ -59,6 +68,20 @@ class TestApprovedBy: assert guard.approved_by(reviews, "kerry-berri") is True assert guard.approved_by(reviews[::-1], "kerry-berri") is False + @pytest.mark.parametrize( + ("states", "approved"), + ( + (("APPROVED", "COMMENTED"), True), + (("APPROVED", "PENDING"), True), + (("CHANGES_REQUESTED", "COMMENTED"), False), + (("APPROVED", "DISMISSED"), False), + (("COMMENTED",), False), + ), + ) + def test_a_comment_or_pending_review_does_not_change_the_answer(self, states, approved): + reviews: Final = tuple(guard.Review(state=state, user=guard.Reviewer(login="kerry-berri")) for state in states) + assert guard.approved_by(reviews, "kerry-berri") is approved + def test_no_review_from_the_approver_is_not_an_approval(self): other: Final = guard.Review(state="APPROVED", user=guard.Reviewer(login="someone-else")) assert guard.approved_by((other,), "kerry-berri") is False From e21de0e418c076be488eca83cf97d2ba3c1998cd Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 28 Oct 2026 15:11:26 -0700 Subject: [PATCH 03/27] fix(logging): bill each SDK call its own native Bedrock guardrail trace when calls share one metadata dict (LIT-10239) (#48923) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 2 +- ...ck_native_guardrail_shared_metadata_sdk.py | 67 +++++++++++++++++++ .../test_litellm_logging.py | 32 +++++++++ 3 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 tests/integration/sdk/test_bedrock_native_guardrail_shared_metadata_sdk.py diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 3b8157b2249..80f1f8f5eca 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2758,7 +2758,7 @@ class Logging(LiteLLMLoggingBaseClass): if entry is None or not is_str_keyed_dict(litellm_params): return metadata: Final[object] = litellm_params.get("metadata") - bucket: Final = metadata if is_str_keyed_dict(metadata) else {} + bucket: Final = {**metadata} if is_str_keyed_dict(metadata) else {} litellm_params["metadata"] = bucket with_guardrail_entry(bucket, entry) diff --git a/tests/integration/sdk/test_bedrock_native_guardrail_shared_metadata_sdk.py b/tests/integration/sdk/test_bedrock_native_guardrail_shared_metadata_sdk.py new file mode 100644 index 00000000000..d0235310aaa --- /dev/null +++ b/tests/integration/sdk/test_bedrock_native_guardrail_shared_metadata_sdk.py @@ -0,0 +1,67 @@ +"""An SDK caller that reuses one ``metadata=`` dict across ``litellm.completion`` calls to a Bedrock deployment whose +``guardrailConfig`` has ``trace: enabled`` is billed, on each call, for that call's guardrail trace only, and the +caller's dict is left as the caller wrote it (LIT-10239)""" + +import json +from collections.abc import Mapping +from typing import Final + +import pytest +from integration._support.client import eventually +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +import litellm + +_MODEL: Final = "bedrock/anthropic.claude-sonnet-4-5-20250929-v1:0" +_PROMPT_TOKENS: Final = 5 +_COMPLETION_TOKENS: Final = 1 +_USAGE: Final[Mapping[str, int]] = {"contentPolicyUnits": 1, "topicPolicyUnits": 1} +_GUARDRAIL_IDS: Final = ("guardrail-one", "guardrail-two") + + +def _bedrock(request: Request) -> Reply: + """Bedrock's Converse runtime: a one-token answer and the trace of the guardrail the request named.""" + guardrail_id: Final = json.loads(request.body)["guardrailConfig"]["guardrailIdentifier"] + body: Final[dict[str, JsonValue]] = { + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": _PROMPT_TOKENS, "outputTokens": _COMPLETION_TOKENS, "totalTokens": 6}, + "metrics": {"latencyMs": 3}, + "trace": {"guardrail": {"inputAssessment": {guardrail_id: {"invocationMetrics": {"usage": dict(_USAGE)}}}}}, + } + return Reply(body=json.dumps(body).encode()) + + +def _guardrail_cost() -> float: + rates: Final = litellm.model_cost["bedrock/guardrails"]["guardrail_cost_per_unit"] + return sum(units * rates[counter] for counter, units in _USAGE.items()) + + +def _llm_cost() -> float: + prices: Final = litellm.model_cost[_MODEL.removeprefix("bedrock/")] + return _PROMPT_TOKENS * prices["input_cost_per_token"] + _COMPLETION_TOKENS * prices["output_cost_per_token"] + + +def test_each_call_sharing_one_metadata_dict_is_billed_its_own_trace_only() -> None: + billed: list[float] = [] # mutable-ok: the success callback appends what each call was billed + litellm.success_callback.append( + lambda kwargs, response, start, end: billed.append(kwargs["standard_logging_object"]["response_cost"]) + ) + shared_metadata: Final = {"tag": "x"} + with wire_server(_bedrock) as wire: + for guardrail_id in _GUARDRAIL_IDS: + litellm.completion( + model=_MODEL, + messages=[{"role": "user", "content": "hello"}], + api_base=wire.url, + aws_access_key_id="AKIASCRIPTED", + aws_secret_access_key="scripted", + aws_region_name="us-east-1", + guardrailConfig={"guardrailIdentifier": guardrail_id, "guardrailVersion": "1", "trace": "enabled"}, + metadata=shared_metadata, + ) + costs: Final = eventually(lambda: tuple(billed), lambda seen: len(seen) == len(_GUARDRAIL_IDS), seconds=30) + + assert costs == pytest.approx((_llm_cost() + _guardrail_cost(),) * len(_GUARDRAIL_IDS)) + assert shared_metadata == {"tag": "x"} diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index e786753e1a8..299088a6c4f 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -13775,3 +13775,35 @@ def test_record_native_bedrock_guardrail_bills_nothing_for_a_response_served_fro logging_obj.record_native_bedrock_guardrail({"trace": trace}) assert "standard_logging_guardrail_information" not in logging_obj.model_call_details["litellm_params"]["metadata"] + + +def test_record_native_bedrock_guardrail_bills_each_call_its_own_trace_when_calls_share_one_metadata_dict() -> None: + import datetime + + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost + + shared_metadata = {"tag": "x"} + traces = tuple( + {"guardrail": {"inputAssessment": {"gid": {"invocationMetrics": {"usage": {"contentPolicyUnits": units}}}}}} + for units in (1, 2) + ) + costs = [] + for call_index, trace in enumerate(traces): + logging_obj = Logging( + model="bedrock/anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=[{"role": "user", "content": "hello"}], + stream=False, + call_type="completion", + start_time=datetime.datetime.now(), + litellm_call_id=f"call-shared-metadata-{call_index}", + function_id="fn", + ) + logging_obj.model_call_details["litellm_params"] = {"metadata": shared_metadata} + logging_obj.record_native_bedrock_guardrail({"trace": trace}) + recorded = logging_obj.model_call_details["litellm_params"]["metadata"]["standard_logging_guardrail_information"] + costs.append(guardrail_information_cost(recorded)) + + (first_cost, second_cost) = costs + assert second_cost == pytest.approx(2 * first_cost) + assert shared_metadata == {"tag": "x"} From 4a70d80aef0c600e4c493a77ab0a69d001d3d3bc Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 28 Oct 2026 15:23:52 -0700 Subject: [PATCH 04/27] fix(aiohttp_transport): keep the owned session open while a response stream is still being read (LIT-10237) (#48920) * fix(aiohttp_transport): keep the owned session open while a response stream is still being read (LIT-10237) * test(aiohttp_transport): annotate the stream regression test locals with Final and drop its docstring --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/custom_httpx/aiohttp_transport.py | 23 +++++++- .../custom_httpx/test_aiohttp_transport.py | 55 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py index 0e4d97fd770..9297e76624d 100644 --- a/litellm/llms/custom_httpx/aiohttp_transport.py +++ b/litellm/llms/custom_httpx/aiohttp_transport.py @@ -6,6 +6,7 @@ import ssl import sys import typing import urllib.request +import weakref from collections.abc import Callable, Generator from typing import ClassVar, Final @@ -109,6 +110,24 @@ def map_aiohttp_exceptions() -> Generator[None, None, None]: raise mapped_exc(message) from exc +def _release_transport_anchor(_transport: object) -> None: + """Release a transport anchored to a response stream. See ``_anchor_transport_to``.""" + + +def _anchor_transport_to(stream: "AiohttpResponseStream", transport: object) -> None: + """Keep the transport alive for as long as the stream can still read. + + The transport finalizer closes the session it owns, and nothing on the path from a + live response back to that session references the transport, so a client dropped + mid-stream would close the session under the body. ``weakref.finalize`` holds the + transport in its own registry rather than on the stream, which keeps the transport + out of the response's reference cycle (an exception carrying the response, for one) + so it is finalized by refcount once the anchor drops and can still schedule an async + close, instead of being collected in the same cyclic pass as the session it closes. + """ + weakref.finalize(stream, _release_transport_anchor, transport) + + class AiohttpResponseStream(httpx.AsyncByteStream): CHUNK_SIZE = 1024 * 16 @@ -463,10 +482,12 @@ class LiteLLMAiohttpTransport(AiohttpTransport): # Re-raise if it's a different RuntimeError raise + stream: Final = AiohttpResponseStream(response) + _anchor_transport_to(stream, self) return httpx.Response( status_code=response.status, headers=response.headers, - stream=AiohttpResponseStream(response), + stream=stream, request=request, ) diff --git a/tests/unit/llms/custom_httpx/test_aiohttp_transport.py b/tests/unit/llms/custom_httpx/test_aiohttp_transport.py index c7d5518ef5b..aed808223bd 100644 --- a/tests/unit/llms/custom_httpx/test_aiohttp_transport.py +++ b/tests/unit/llms/custom_httpx/test_aiohttp_transport.py @@ -5,6 +5,7 @@ import socket import sys import struct import threading +import weakref from typing import Final import aiohttp @@ -1383,3 +1384,57 @@ def test_a_transport_collected_while_its_loop_runs_elsewhere_closes_its_session_ loop.call_soon_threadsafe(loop.stop) thread.join(timeout=10) loop.close() + + +@pytest.mark.asyncio +async def test_a_stream_keeps_reading_after_the_client_that_opened_it_is_collected(): + from aiohttp import web + + release: Final = asyncio.Event() + body: Final = tuple(f"chunk{i}\n".encode() for i in range(4)) + + async def handler(request: web.Request) -> web.StreamResponse: + response: Final = web.StreamResponse() + await response.prepare(request) + await response.write(body[0]) + await release.wait() + for chunk in body[1:]: + await response.write(chunk) + await response.write_eof() + return response + + app: Final = web.Application() + app.router.add_get("/stream", handler) + runner: Final = web.AppRunner(app) + await runner.setup() + sock: Final = socket.socket() + sock.bind(("127.0.0.1", 0)) + port: Final[int] = sock.getsockname()[1] + await web.SockSite(runner, sock).start() + + transport: Final = LiteLLMAiohttpTransport(client=aiohttp.ClientSession) + client: Final = httpx.AsyncClient(transport=transport) + transport_ref: Final = weakref.ref(transport) + try: + response: Final = await client.send(client.build_request("GET", f"http://127.0.0.1:{port}/stream"), stream=True) + session: Final = transport.client + assert isinstance(session, aiohttp.ClientSession) + chunks: Final = response.aiter_bytes() + first: Final = await chunks.__anext__() + + del client, transport + gc.collect() + await asyncio.gather(*LiteLLMAiohttpTransport._background_close_tasks) + release.set() + + rest: Final = [chunk async for chunk in chunks] + assert first + b"".join(rest) == b"".join(body), (first, rest) + assert not session.closed, "the session was closed under a stream that was still being read" + + del response, chunks + gc.collect() + await asyncio.gather(*LiteLLMAiohttpTransport._background_close_tasks) + assert transport_ref() is None, "the finished stream kept the transport alive" + assert session.closed, "the owned session stayed open after its last stream finished" + finally: + await runner.cleanup() From 2791822f921fab9469b86ed37d8a2deadad0fc58 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 28 Oct 2026 15:28:35 -0700 Subject: [PATCH 05/27] ci(spend-logs-guard): run only on pull_request_target, re-run by hand after approving (#48941) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/spend-logs-guard.yml | 4 ---- ci_cd/spend_logs_guard.py | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/spend-logs-guard.yml b/.github/workflows/spend-logs-guard.yml index 354a1b7c947..29e9f236202 100644 --- a/.github/workflows/spend-logs-guard.yml +++ b/.github/workflows/spend-logs-guard.yml @@ -4,10 +4,6 @@ on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the P pull_request_target: branches: - main - pull_request_review: - types: - - submitted - - dismissed permissions: contents: read diff --git a/ci_cd/spend_logs_guard.py b/ci_cd/spend_logs_guard.py index fac9d4ba5d2..a4d9a2e0a0d 100644 --- a/ci_cd/spend_logs_guard.py +++ b/ci_cd/spend_logs_guard.py @@ -6,7 +6,7 @@ so a path rule would put every migration on the owner. This gate reads the diff added line in the watched files mentions the table it passes at once, otherwise it passes only once the owner's latest review on the pull request is an approval that still covers every spend-log line in the head (approvals survive pushes, so a later commit may not add spend-log lines the owner never -saw). The workflow re-runs on every review. +saw). Reviews do not trigger the workflow, so the check is re-run by hand after approving. """ from __future__ import annotations @@ -153,7 +153,7 @@ def main(argv: Sequence[str]) -> int: else f"LiteLLM_SpendLogs lines changed after @{args.approver} approved {approval.commit_id}, " f"a new approving review is needed:\n" ) - sys.stdout.write(f"{reason}{listed}The check re-runs when a review is submitted.\n") + sys.stdout.write(f"{reason}{listed}After approving, re-run this job.\n") return 1 From 84c4c5aa5c530610863afe104b32dc50ebca37da Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 28 Oct 2026 15:32:27 -0700 Subject: [PATCH 06/27] fix(router): cool down failing deployments when fallbacks use string targets (LIT-10238) (#48913) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router_utils/cooldown_handlers.py | 32 ++++++++++++++----- .../sdk/test_router_cooldown_fallback_sdk.py | 28 ++++++++++++++++ .../router_utils/test_cooldown_handlers.py | 32 +++++++++++++++++++ 3 files changed, 84 insertions(+), 8 deletions(-) diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 128bca6cca0..a1b1e327802 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -13,7 +13,7 @@ from datetime import datetime from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError import litellm from litellm._internal_context import service_target @@ -417,17 +417,35 @@ def _should_run_cooldown_logic( return True -_FALLBACK_ENTRIES_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, list[str | Mapping[str, object]]] | str, ...]) +_FALLBACK_ENTRIES_ADAPTER: Final = TypeAdapter(tuple[object, ...]) +_FALLBACK_ENTRY_ADAPTER: Final = TypeAdapter(Mapping[str, list[str | Mapping[str, object]] | str]) def _fallback_target_name(target: str | Mapping[str, object]) -> str: return target if isinstance(target, str) else str(target.get("model", "")) +def _fallback_target_names( + targets: Sequence[str | Mapping[str, object]] | str, +) -> list[str]: # mutable-ok: mirrors get_fallback_model_group's fallbacks shape + return [targets] if isinstance(targets, str) else [_fallback_target_name(target) for target in targets] + + def _fallback_chain_names( - entry: Mapping[str, Sequence[str | Mapping[str, object]]], -) -> Mapping[str, list[str]]: # mutable-ok: mirrors get_fallback_model_group's fallbacks shape - return {key: [_fallback_target_name(target) for target in targets] for key, targets in entry.items()} + entry: object, +) -> Mapping[str, list[str]] | None: # mutable-ok: mirrors get_fallback_model_group's fallbacks shape + try: + validated: Final = _FALLBACK_ENTRY_ADAPTER.validate_python(entry) + except ValidationError: + return None + return {key: _fallback_target_names(targets) for key, targets in validated.items()} + + +def _fallback_entries(configured: object) -> tuple[object, ...]: + try: + return _FALLBACK_ENTRIES_ADAPTER.validate_python(configured) + except ValidationError: + return () def _has_fallback_group(litellm_router_instance: LitellmRouter, requested_model_group: str | None) -> bool: @@ -437,9 +455,7 @@ def _has_fallback_group(litellm_router_instance: LitellmRouter, requested_model_ if requested_model_group is None or not configured: return False chains: Final = tuple( - _fallback_chain_names(entry) - for entry in _FALLBACK_ENTRIES_ADAPTER.validate_python(configured) - if isinstance(entry, Mapping) + chain for chain in map(_fallback_chain_names, _fallback_entries(configured)) if chain is not None ) fallback_group, _ = get_fallback_model_group(fallbacks=chains, model_group=requested_model_group) return bool(fallback_group) diff --git a/tests/integration/sdk/test_router_cooldown_fallback_sdk.py b/tests/integration/sdk/test_router_cooldown_fallback_sdk.py index 747adeee41d..e7508a2670e 100644 --- a/tests/integration/sdk/test_router_cooldown_fallback_sdk.py +++ b/tests/integration/sdk/test_router_cooldown_fallback_sdk.py @@ -12,6 +12,7 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm import Router +from litellm.router_utils.cooldown_handlers import async_get_cooldown_deployments from litellm.types.router import DeploymentTypedDict _MODEL: Final = "gpt-5.4" @@ -118,3 +119,30 @@ async def test_cooldown_skips_only_the_failed_duplicate_model_deployment() -> No "/primary/chat/completions", "/backup/chat/completions", ) + + +@pytest.mark.parametrize( + "router_kwargs", + [{"fallbacks": [{"primary": "backup"}]}, {"default_fallbacks": "backup"}], + ids=["string_fallback_target", "string_default_fallbacks"], +) +@pytest.mark.asyncio +async def test_string_valued_fallbacks_still_cool_down_the_failing_primary(router_kwargs: dict[str, object]) -> None: + with wire_server(_peer) as wire: + router: Final = Router( + model_list=[ + _deployment("primary", f"{wire.url}/primary", "primary-deployment"), + _deployment("backup", f"{wire.url}/backup", "backup-deployment"), + ], + allowed_fails=0, + num_retries=0, + **router_kwargs, + ) + with pytest.raises(litellm.AuthenticationError): + await router.acompletion( + model="primary", + messages=[{"role": "user", "content": f"cooldown {uuid.uuid4().hex}"}], + ) + cooled: Final = await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) + + assert tuple(cooled) == ("primary-deployment",) diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 0d39016ea79..24459a5a250 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -25,6 +25,7 @@ from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callba from litellm.router_utils.cooldown_handlers import ( _get_deployment_cooldown_policy, _has_explicit_allowed_fails_policy_for_exception, + _has_fallback_group, _increment_allowed_fails, _is_cooldown_required, _resolve_allowed_fails_from_policy, @@ -3149,6 +3150,8 @@ BENCHED: Final = (litellm.Timeout, litellm.Timeout, RouterRateLimitError, Router {}, BENCHED, ), + ({"allowed_fails": 1, "fallbacks": [{"test-model": "backup-model"}]}, {}, BENCHED), + ({"allowed_fails": 1, "default_fallbacks": "backup-model"}, {}, BENCHED), ], ids=[ "allowed_fails", @@ -3157,6 +3160,8 @@ BENCHED: Final = (litellm.Timeout, litellm.Timeout, RouterRateLimitError, Router "policy_names_timeout", "allowed_fails_with_a_fallback_group", "allowed_fails_with_a_dict_fallback_target", + "allowed_fails_with_a_string_fallback_target", + "allowed_fails_with_a_string_default_fallbacks", ], ) @pytest.mark.asyncio @@ -3180,6 +3185,33 @@ async def test_router_allowed_fails_benches_a_lone_deployment_only_when_the_poli assert len(route.calls) == outcomes.count(litellm.Timeout) +@pytest.mark.parametrize( + ("fallbacks", "expected"), + [ + ([{"test-model": "backup-model"}], True), + ([{"*": "backup-model"}], True), + ([{"test-model": ["backup-model"]}], True), + ([{"test-model": 5}, {"*": "backup-model"}], True), + ([{"test-model": 5}], False), + ([{"other-model": ["backup-model"]}], False), + ], + ids=[ + "string_target", + "string_catch_all_from_default_fallbacks", + "list_target", + "unparseable_entry_does_not_hide_the_catch_all", + "unparseable_entry_alone", + "another_groups_chain", + ], +) +def test_has_fallback_group_reads_string_targets_and_skips_unparseable_entries( + fallbacks: Sequence[Mapping[str, object]], expected: bool +) -> None: + router: Final = Router(model_list=[TIMED_OUT_DEPLOYMENT], fallbacks=list(fallbacks)) + + assert _has_fallback_group(router, "test-model") is expected + + async def _group_call_outcome(router: Router) -> str: try: response: Final = await router.acompletion( From e157c5e520acbd83ed75bb26ec3633c4b2cc09b1 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 28 Oct 2026 15:34:15 -0700 Subject: [PATCH 07/27] fix(migration_data_checks): size btree entries by the bytes Postgres stores (LIT-10244) (#48911) * fix(migration_data_checks): size btree entries by the bytes Postgres stores (LIT-10244) * test(migration_data_checks): flag exactly the compressed keys Postgres rejects at the entry limit (LIT-10244) --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../migration_data_checks.py | 11 +++- .../migrations/test_request_log_indexes.py | 64 +++++++++++++++++++ .../test_migration_data_checks.py | 15 +++++ 3 files changed, 87 insertions(+), 3 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py b/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py index 7b01e64dd73..e22639daaef 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py +++ b/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py @@ -343,11 +343,16 @@ def btree_max_entry_bytes(block_size: int) -> int: def _value_bytes(column: ColumnType, identifier: "sql.Identifier") -> "sql.Composed": + """The bytes a value takes in the index entry: its length, or its stored size when + Postgres compressed it, since `index_form_tuple` keeps the compressed bytes.""" from psycopg import sql - if column.type_name in _OCTET_LENGTH_TYPES: - return sql.SQL("octet_length({})").format(identifier) - return sql.SQL("octet_length({}::text)").format(identifier) + length: Final = ( + sql.SQL("octet_length({})").format(identifier) + if column.type_name in _OCTET_LENGTH_TYPES + else sql.SQL("octet_length({}::text)").format(identifier) + ) + return sql.SQL("least({}, pg_column_size({}))").format(length, identifier) def _aligned(offset: "sql.Composable", align: str) -> "sql.Composed": diff --git a/tests/integration/database/migrations/test_request_log_indexes.py b/tests/integration/database/migrations/test_request_log_indexes.py index 1de2090b487..f558c355c84 100644 --- a/tests/integration/database/migrations/test_request_log_indexes.py +++ b/tests/integration/database/migrations/test_request_log_indexes.py @@ -1,3 +1,4 @@ +import hashlib import os import queue import secrets @@ -973,6 +974,7 @@ def test_the_migration_job_refuses_before_the_first_migration_when_spend_log_key _insert_spend_log_with_key(conn, "legacy-1", _legacy_bearer_token()) _insert_spend_log_with_key(conn, "legacy-2", _legacy_bearer_token()) _insert_spend_log_with_key(conn, "hashed-1", "a" * 64) + _insert_spend_log_with_key(conn, "repetitive-1", "z" * 3000) ledger_before: Final = _ledger(scratch_database) with pytest.raises(RuntimeError) as refused: @@ -1109,6 +1111,68 @@ def test_the_entry_size_check_draws_the_line_exactly_where_postgres_does( assert finding.detail == f"with an index entry over 2704 bytes, largest 2712 bytes ({described_as})" +@requires_db +def test_a_value_postgres_compresses_under_the_limit_is_not_counted_while_random_text_of_that_length_is( + scratch_database: str, +) -> None: + """Postgres compresses a long value before it checks the entry size, so 3000 repeated + bytes index fine while 3000 random hex bytes do not. The check must draw the same line.""" + create_index: Final = sql.SQL('CREATE INDEX "compress_probe_idx" ON "compress_probe" ("k")') + with psycopg.connect(scratch_database, autocommit=True) as conn: + conn.execute('CREATE TABLE "compress_probe" ("k" TEXT)') + conn.execute('INSERT INTO "compress_probe" ("k") VALUES (%s), (%s)', ("a" * 3000, secrets.token_hex(1500))) + findings: Final = migration_data_checks.find_blockers( + conn, "public", migration_data_checks.checks_for_statement(create_index.as_string(conn), "probe") + ) + with pytest.raises(psycopg.errors.ProgramLimitExceeded, match="index row size 3016 exceeds"): + conn.execute(create_index) + conn.execute('DELETE FROM "compress_probe" WHERE "k" <> %s', ("a" * 3000,)) + conn.execute(create_index) + stored: Final = conn.execute('SELECT octet_length("k"), pg_column_size("k") FROM "compress_probe"').fetchone() + + assert stored is not None and stored[0] == 3000 and stored[1] < 2704, stored + (finding,) = findings + assert isinstance(finding, migration_data_checks.BlockedChange) + assert finding.rows == 1, "only the random key is too long for the index once Postgres compresses the other" + assert finding.detail == "with an index entry over 2704 bytes, largest 3016 bytes (k up to 3000 bytes)" + + +def _postgres_rejects_an_index_on(conn: "psycopg.Connection[tuple[object, ...]]", key: str) -> bool: + conn.execute('DROP TABLE IF EXISTS "one_key"') + conn.execute('CREATE TABLE "one_key" ("k" TEXT)') + conn.execute('INSERT INTO "one_key" ("k") VALUES (%s)', (key,)) + try: + conn.execute('CREATE INDEX "one_key_idx" ON "one_key" ("k")') + except psycopg.errors.ProgramLimitExceeded: + return True + return False + + +@requires_db +def test_the_check_flags_exactly_the_compressed_keys_postgres_rejects_at_the_entry_limit(scratch_database: str) -> None: + """Keys that compress to a stored size just under and just over the entry limit go through the + check and through Postgres one by one. The check must flag a key exactly when Postgres rejects it.""" + noise: Final = "".join(hashlib.sha256(str(i).encode()).hexdigest() for i in range(40)) + keys: Final = tuple(noise[:n] + "a" * 1500 for n in range(2400, 2500)) + create_index: Final = sql.SQL('CREATE INDEX "stored_probe_idx" ON "stored_probe" ("k")') + with psycopg.connect(scratch_database, autocommit=True) as conn: + conn.execute('CREATE TABLE "stored_probe" ("k" TEXT)') + conn.execute('INSERT INTO "stored_probe" ("k") SELECT unnest(%s::text[])', (list(keys),)) + compressed: Final = conn.execute( + 'SELECT count(*) FROM "stored_probe" WHERE pg_column_size("k") < octet_length("k") AND octet_length("k") > 2704' + ).fetchone() + findings: Final = migration_data_checks.find_blockers( + conn, "public", migration_data_checks.checks_for_statement(create_index.as_string(conn), "probe") + ) + rejected: Final = tuple(key for key in keys if _postgres_rejects_an_index_on(conn, key)) + + assert compressed == (len(keys),), "every probe key must be longer than the limit yet stored compressed" + assert 0 < len(rejected) < len(keys), f"the probe must straddle the limit, Postgres rejected {len(rejected)}" + (finding,) = findings + assert isinstance(finding, migration_data_checks.BlockedChange) + assert finding.rows == len(rejected), f"the check must flag the {len(rejected)} keys Postgres rejects and no other" + + @requires_db def test_a_scan_that_cannot_finish_in_time_is_reported_as_unverified_rather_than_passed(scratch_database: str) -> None: with psycopg.connect(scratch_database, autocommit=True) as conn: diff --git a/tests/unit/litellm_proxy_extras/test_migration_data_checks.py b/tests/unit/litellm_proxy_extras/test_migration_data_checks.py index f3b787beda1..d4490927e29 100644 --- a/tests/unit/litellm_proxy_extras/test_migration_data_checks.py +++ b/tests/unit/litellm_proxy_extras/test_migration_data_checks.py @@ -7,7 +7,9 @@ from litellm_proxy_extras.migration_data_checks import ( MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR, BlockedChange, Check, + ColumnType, CouldNotVerify, + _count_query, btree_max_entry_bytes, checks_for_index, checks_for_migration, @@ -113,6 +115,19 @@ def test_the_btree_entry_limit_on_the_default_page_is_the_number_in_postgres_err assert btree_max_entry_bytes(8192) == 2704 +def test_the_entry_size_query_sizes_a_text_column_by_its_stored_bytes_when_postgres_compressed_it() -> None: + """Postgres compresses a long value before it checks the entry size, so a repetitive + 3000-byte key indexes fine. The query has to size it as stored, never by its length alone.""" + (check,) = checks_for_index( + "LiteLLM_SpendLogs", "LiteLLM_SpendLogs_api_key_startTime_idx", ("api_key", "startTime") + ) + types: Final = (ColumnType("text", -1, "i", -1), ColumnType("timestamp", 8, "d", 3)) + query: Final = _count_query(check, "public", types, 2704).as_string(None) + stored: Final = 'least(octet_length("api_key"), pg_column_size("api_key"))' + assert stored in query + assert 'octet_length("api_key")' not in query.replace(stored, ""), query + + def test_the_report_names_the_rows_the_statement_and_the_fix_and_says_nothing_was_applied() -> None: (check,) = checks_for_index( "LiteLLM_SpendLogs", "LiteLLM_SpendLogs_api_key_startTime_idx", ("api_key", "startTime") From 0b57b9976a609ab74ba5c5cf21f75079527527ff Mon Sep 17 00:00:00 2001 From: mrinal-berri Date: Wed, 28 Oct 2026 15:35:08 -0700 Subject: [PATCH 08/27] feat(daily-activity): per-guardrail spend breakdown in user, team and organization daily activity (#48702) * feat(daily-activity): read guardrail spend per key from the daily guardrail usage table Adds DailyActivityRepository.guardrail_spend, which sums LiteLLM_DailyGuardrailUsageUnits.cost by date, guardrail and key. A guardrail row counts only when its key has a spend row on that day under the same scope filters as the rest of the daily activity response, so user, key and date filters apply to guardrail cost too * feat(daily-activity): add a per-guardrail spend breakdown to /user/daily/activity/aggregated Each day now has breakdown.guardrails[] with metrics.spend and an api_key_breakdown carrying each key's spend and metadata. Guardrail entries only report spend, since that is the one metric stored per key. The breakdown is on the user endpoint only, is empty when a model filter is set because guardrail cost is not stored per model, and lists only keys within api_key_limit while the guardrail total keeps every key. spend and budgets are unchanged Resolves LIT-9785 * feat(daily-activity): add the guardrail breakdown to team and organization activity, with guardrail names /team/daily/activity/aggregated and /organization/daily/activity/aggregated now return breakdown.guardrails the same way the user endpoint does, since each key belongs to one team and one organization. Each guardrail entry also carries metadata.guardrail_name, read from LiteLLM_GuardrailsTable for guardrails created through the UI or API and falling back to the breakdown key for config.yaml guardrails, whose key is already their name * fix(daily-activity): keep daily activity up when the guardrail breakdown query fails, and treat an empty model filter as none A database error from the guardrail spend query now logs a warning and returns the response without guardrails, where it used to fail the whole aggregated response with a 500. An empty model= no longer drops the guardrail breakdown, matching how the rest of the endpoint already ignores it. The daily activity golden files and fixtures gain the empty guardrails key every breakdown now carries * refactor(daily-activity): fold guardrail breakdown in a worker thread and drop docstrings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(daily-activity): report untracked guardrail units next to spend Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_lazy_openapi_snapshot.json | 75 ++++++++ .../common_daily_activity.py | 68 ++++++- .../daily_activity_routes.py | 1 + .../daily_activity_scopes.py | 4 + .../repositories/daily_activity_repository.py | 6 + litellm/repositories/daily_activity_sql.py | 19 ++ .../common_daily_activity.py | 29 +++ litellm/types/repositories/daily_activity.py | 10 + .../spend/fixtures/daily_activity_team.json | 2 +- .../spend/fixtures/daily_activity_user.json | 2 +- .../daily_activity_team_aggregated.json | 2 +- .../golden/daily_activity_team_paginated.json | 2 +- .../daily_activity_user_aggregated.json | 2 +- .../golden/daily_activity_user_paginated.json | 2 +- ...st_daily_activity_aggregated_breakdowns.py | 180 ++++++++++++++++++ .../test_daily_activity_routes.py | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 36 ++++ 17 files changed, 432 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index b10b5ae98b5..39adb99a0d0 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -3487,6 +3487,13 @@ "title": "Entities", "type": "object" }, + "guardrails": { + "additionalProperties": { + "$ref": "#/components/schemas/GuardrailMetric" + }, + "title": "Guardrails", + "type": "object" + }, "mcp_servers": { "additionalProperties": { "$ref": "#/components/schemas/MetricWithMetadata" @@ -3802,6 +3809,74 @@ "title": "ExportType", "type": "string" }, + "GuardrailKeyMetric": { + "properties": { + "metadata": { + "$ref": "#/components/schemas/KeyMetadata" + }, + "metrics": { + "$ref": "#/components/schemas/GuardrailSpendMetrics" + } + }, + "required": [ + "metrics", + "metadata" + ], + "title": "GuardrailKeyMetric", + "type": "object" + }, + "GuardrailMetadata": { + "properties": { + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + } + }, + "required": [ + "guardrail_name" + ], + "title": "GuardrailMetadata", + "type": "object" + }, + "GuardrailMetric": { + "properties": { + "api_key_breakdown": { + "additionalProperties": { + "$ref": "#/components/schemas/GuardrailKeyMetric" + }, + "title": "Api Key Breakdown", + "type": "object" + }, + "metadata": { + "$ref": "#/components/schemas/GuardrailMetadata" + }, + "metrics": { + "$ref": "#/components/schemas/GuardrailSpendMetrics" + } + }, + "required": [ + "metrics", + "metadata" + ], + "title": "GuardrailMetric", + "type": "object" + }, + "GuardrailSpendMetrics": { + "properties": { + "spend": { + "default": 0.0, + "title": "Spend", + "type": "number" + }, + "untracked_units": { + "default": 0, + "title": "Untracked Units", + "type": "integer" + } + }, + "title": "GuardrailSpendMetrics", + "type": "object" + }, "HTTPAuthSecurityScheme": { "description": "Defines a security scheme using HTTP authentication.", "properties": { diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index f948a49441a..66f386b792d 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -3,6 +3,7 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass, replace from datetime import date, datetime, timedelta +from itertools import groupby from types import MappingProxyType from typing import Final, Literal, NoReturn, Protocol @@ -28,6 +29,10 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailySpendData, DailySpendMetadata, GroupedData, + GuardrailKeyMetric, + GuardrailMetadata, + GuardrailMetric, + GuardrailSpendMetrics, KeyMetadata, KeyMetricWithMetadata, MetricWithMetadata, @@ -39,6 +44,7 @@ from litellm.types.repositories.daily_activity import ( DailyActivityTable, EntityRollupRow, GroupingSetsRow, + GuardrailSpendRow, KeyMetadataRow, RollupMetricsRow, SpendLogsWindow, @@ -1013,17 +1019,68 @@ def _fold_entity_rollups_sync( ) +def _guardrail_metric(day: DailySpendData, rows: Sequence[GuardrailSpendRow]) -> GuardrailMetric: + return GuardrailMetric( + metrics=GuardrailSpendMetrics( + spend=sum(row.spend for row in rows), + untracked_units=sum(row.untracked_units for row in rows), + ), + metadata=GuardrailMetadata(guardrail_name=rows[0].guardrail_name), + api_key_breakdown={ + row.api_key: GuardrailKeyMetric( + metrics=GuardrailSpendMetrics(spend=row.spend, untracked_units=row.untracked_units), + metadata=day.breakdown.api_keys[row.api_key].metadata, + ) + for row in rows + if row.api_key in day.breakdown.api_keys + }, + ) + + +def _day_with_guardrails(day: DailySpendData, rows: Sequence[GuardrailSpendRow]) -> DailySpendData: + by_guardrail: Final = groupby(sorted(rows, key=lambda row: row.guardrail_id), key=lambda row: row.guardrail_id) + guardrails: Final = {guardrail_id: _guardrail_metric(day, tuple(group)) for guardrail_id, group in by_guardrail} + return day.model_copy(update={"breakdown": day.breakdown.model_copy(update={"guardrails": guardrails})}) + + +def _with_guardrail_breakdown( + results: Sequence[DailySpendData], rows: Sequence[GuardrailSpendRow] +) -> tuple[DailySpendData, ...]: + rows_by_date: Final = MappingProxyType( + {day: tuple(group) for day, group in groupby(sorted(rows, key=lambda row: row.date), key=lambda row: row.date)} + ) + return tuple(_day_with_guardrails(day, rows_by_date.get(day.date.strftime("%Y-%m-%d"), ())) for day in results) + + +async def _guardrail_spend_or_empty( + repository: DailyActivityRepository, scope: DailyActivityScope +) -> tuple[GuardrailSpendRow, ...]: + from prisma.errors import PrismaError + + try: + return await repository.guardrail_spend(scope) + except PrismaError as exc: + verbose_proxy_logger.warning("Returning daily activity without its guardrail breakdown: %s", exc) + return () + + async def get_daily_activity_aggregated( repository: DailyActivityRepository, scope: DailyActivityScope, *, entity_metadata_field: Mapping[str, dict[str, object]] | None = None, include_entity_breakdown: bool = False, + include_guardrail_breakdown: bool = False, api_key_limit: int = constants.USAGE_TOP_API_KEYS_DEFAULT, ) -> SpendAnalyticsPaginatedResponse: try: - aggregated_rows: Final = await repository.aggregated( - scope, include_entity_breakdown=include_entity_breakdown, api_key_limit=api_key_limit + aggregated_rows, guardrail_rows = await asyncio.gather( + repository.aggregated( + scope, include_entity_breakdown=include_entity_breakdown, api_key_limit=api_key_limit + ), + _guardrail_spend_or_empty(repository, scope) + if include_guardrail_breakdown and not scope.model + else asyncio.sleep(0, result=()), ) records: Final = aggregated_rows.grouping_rows aggregated: Final = await _aggregate_grouping_sets_records( @@ -1058,8 +1115,13 @@ async def get_daily_activity_aggregated( api_key_metadata=entity_key_metadata, entity_metadata_field=entity_metadata_field, ) + results: Final = ( + list(await asyncio.to_thread(_with_guardrail_breakdown, aggregated["results"], guardrail_rows)) + if guardrail_rows + else aggregated["results"] + ) return SpendAnalyticsPaginatedResponse( - results=aggregated["results"], + results=results, metadata=DailySpendMetadata( total_spend=aggregated["totals"].spend, total_flat_cost=aggregated["totals"].flat_cost, diff --git a/litellm/proxy/management_endpoints/daily_activity_routes.py b/litellm/proxy/management_endpoints/daily_activity_routes.py index 842873129e4..79e515944ce 100644 --- a/litellm/proxy/management_endpoints/daily_activity_routes.py +++ b/litellm/proxy/management_endpoints/daily_activity_routes.py @@ -300,6 +300,7 @@ def _register_aggregated_route(router: APIRouter, resolver: EntityScopeResolver, resolved.scope, entity_metadata_field=resolved.entity_metadata, include_entity_breakdown=resolver.include_entity_breakdown, + include_guardrail_breakdown=resolver.include_guardrail_breakdown, api_key_limit=api_key_limit, ) if resolver.entity != "team": diff --git a/litellm/proxy/management_endpoints/daily_activity_scopes.py b/litellm/proxy/management_endpoints/daily_activity_scopes.py index bc373388a4e..77246f5dda6 100644 --- a/litellm/proxy/management_endpoints/daily_activity_scopes.py +++ b/litellm/proxy/management_endpoints/daily_activity_scopes.py @@ -56,6 +56,7 @@ class EntityScopeResolver: resolve: EntityScopeResolve include_entity_breakdown: bool operation_names: OperationNames + include_guardrail_breakdown: bool = False def _query_ids(value: str | None) -> tuple[str, ...] | None: @@ -364,6 +365,7 @@ USER_RESOLVER = EntityScopeResolver( query=_user_query, resolve=_resolve_user, include_entity_breakdown=False, + include_guardrail_breakdown=True, operation_names=MappingProxyType( { "aggregated": "get_user_daily_activity_aggregated", @@ -385,6 +387,7 @@ TEAM_RESOLVER = EntityScopeResolver( query=_team_query, resolve=_resolve_team, include_entity_breakdown=True, + include_guardrail_breakdown=True, operation_names=MappingProxyType( { "aggregated": "get_team_daily_activity_aggregated", @@ -423,6 +426,7 @@ ORGANIZATION_RESOLVER = EntityScopeResolver( query=_organization_query, resolve=_resolve_organization, include_entity_breakdown=True, + include_guardrail_breakdown=True, operation_names=MappingProxyType( { "aggregated": "get_organization_daily_activity_aggregated", diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 10c603fe56f..4ca23e4e89e 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -19,6 +19,7 @@ from litellm.repositories.daily_activity_sql import ( build_cache_leakage_keys_sql, build_entity_rollup_sql, build_export_sql, + build_guardrail_spend_sql, build_key_page_sql, build_key_search_sql, build_model_top_keys_sql, @@ -36,6 +37,7 @@ from litellm.types.repositories.daily_activity import ( ExportRow, ExportType, GroupingSetsRow, + GuardrailSpendRow, KeyMetadataRow, KeyPage, KeySpendRow, @@ -100,6 +102,7 @@ class DailyActivityDatabase(Protocol): _GROUPING_ADAPTER: Final = TypeAdapter(tuple[GroupingSetsRow, ...]) _ENTITY_ADAPTER: Final = TypeAdapter(tuple[EntityRollupRow, ...]) +_GUARDRAIL_SPEND_ADAPTER: Final = TypeAdapter(tuple[GuardrailSpendRow, ...]) _KEY_SPEND_ADAPTER: Final = TypeAdapter(tuple[KeySpendRow, ...]) _KEY_PAGE_TOTAL_ADAPTER: Final[TypeAdapter[int]] = TypeAdapter(int) _USER_SPEND_ADAPTER: Final = TypeAdapter(tuple[UserSpendRow, ...]) @@ -191,6 +194,9 @@ class DailyActivityRepository: distinct_api_keys=distinct_api_keys, ) + async def guardrail_spend(self, scope: DailyActivityScope) -> tuple[GuardrailSpendRow, ...]: + return _GUARDRAIL_SPEND_ADAPTER.validate_python(await self._query(build_guardrail_spend_sql(scope))) + async def search_keys(self, scope: DailyActivityScope, *, search: str, limit: int) -> tuple[str, ...]: if not 1 <= limit <= constants.USAGE_KEY_SEARCH_MAX: raise ValueError(f"limit must be between 1 and {constants.USAGE_KEY_SEARCH_MAX}") diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index 3d27e7d937e..7ec9fa67a67 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -261,6 +261,25 @@ def build_entity_rollup_sql(scope: DailyActivityScope, *, api_key_limit: int) -> return SqlQuery(sql=sql, params=(*where_params, PTU_SENTINEL_API_KEY, api_key_limit)) +def build_guardrail_spend_sql(scope: DailyActivityScope) -> SqlQuery: + pg_table: Final = PRISMA_TO_PG_TABLE[scope.table] + where_clause, where_params = build_where_clause(scope) + sql: Final = f""" + SELECT g.date, g.guardrail_id, COALESCE(t.guardrail_name, g.guardrail_id) AS guardrail_name, g.api_key, + COALESCE(SUM(g.cost), 0)::float AS spend, + COALESCE(SUM(CASE WHEN g.cost IS NULL THEN g.units ELSE g.untracked_units END), 0)::bigint AS untracked_units + FROM "LiteLLM_DailyGuardrailUsageUnits" g + LEFT JOIN "LiteLLM_GuardrailsTable" t ON t.guardrail_id = g.guardrail_id + WHERE g.date >= $1 AND g.date <= $2 + AND EXISTS ( + SELECT 1 FROM "{pg_table}" + WHERE {where_clause} AND "{pg_table}".date = g.date AND "{pg_table}".api_key = g.api_key + ) + GROUP BY g.date, g.guardrail_id, t.guardrail_name, g.api_key + """ + return SqlQuery(sql=sql, params=where_params) + + def _key_spend_select() -> str: return """ COALESCE(SUM(spend), 0)::float AS spend, diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 213c911cfdc..3f1a7980b27 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -65,6 +65,34 @@ class MetricWithMetadata(MetricBase): api_key_breakdown: dict[str, KeyMetricWithMetadata] = Field(default_factory=dict) # api_key -> {metrics, metadata} +class GuardrailSpendMetrics(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + spend: float = 0.0 + untracked_units: int = 0 + + +class GuardrailKeyMetric(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + metrics: GuardrailSpendMetrics + metadata: KeyMetadata + + +class GuardrailMetadata(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + guardrail_name: str + + +class GuardrailMetric(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + metrics: GuardrailSpendMetrics + metadata: GuardrailMetadata + api_key_breakdown: Mapping[str, GuardrailKeyMetric] = Field(default_factory=dict) + + class BreakdownMetrics(LiteLLMBaseModel): """Breakdown of spend by different dimensions""" @@ -75,6 +103,7 @@ class BreakdownMetrics(LiteLLMBaseModel): endpoints: dict[str, MetricWithMetadata] = Field(default_factory=dict) # endpoint -> {metrics, metadata} api_keys: dict[str, KeyMetricWithMetadata] = Field(default_factory=dict) # api_key -> {metrics, metadata} entities: dict[str, MetricWithMetadata] = Field(default_factory=dict) # entity -> {metrics, metadata} + guardrails: Mapping[str, GuardrailMetric] = Field(default_factory=dict) # guardrail_id -> spend class DailySpendData(LiteLLMBaseModel): diff --git a/litellm/types/repositories/daily_activity.py b/litellm/types/repositories/daily_activity.py index 4111b83a891..b013ce43113 100644 --- a/litellm/types/repositories/daily_activity.py +++ b/litellm/types/repositories/daily_activity.py @@ -168,6 +168,16 @@ class EntityRollupRow(RollupMetricsRow): distinct_api_keys: int | None +@dataclass(frozen=True, slots=True) +class GuardrailSpendRow: + date: str + guardrail_id: str + guardrail_name: str + api_key: str + spend: float + untracked_units: int + + @dataclass(frozen=True, slots=True) class AggregatedRows: grouping_rows: tuple[GroupingSetsRow, ...] diff --git a/tests/integration/spend/fixtures/daily_activity_team.json b/tests/integration/spend/fixtures/daily_activity_team.json index 62377c07432..8f4956dd7fe 100644 --- a/tests/integration/spend/fixtures/daily_activity_team.json +++ b/tests/integration/spend/fixtures/daily_activity_team.json @@ -1 +1 @@ -{"results":[{"date":"2026-06-01","metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"breakdown":{"mcp_servers":{},"models":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"model_groups":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"providers":{"provider-a":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"endpoints":{"/v1/chat/completions":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"api_keys":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}},"entities":{"team-1":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{"team_alias":"Usage Team"},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}}}}}],"metadata":{"total_spend":1273.0,"total_flat_cost":0.0,"total_ptu_hours":0.0,"total_prompt_tokens":1014,"total_completion_tokens":10,"total_tokens":1024,"total_api_requests":5,"total_successful_requests":5,"total_failed_requests":0,"total_cache_read_input_tokens":6,"total_cache_creation_input_tokens":0,"total_compression_saved_tokens":0,"total_compression_savings_spend":0.0,"total_prompt_caching_savings_spend":0.0,"total_gateway_injected_caching_savings_spend":0.0,"total_autorouter_savings_spend":0.0,"total_response_time_ms":0,"total_timed_requests":0,"page":1,"total_pages":1,"has_more":false,"api_key_limit":100,"total_api_keys":5,"entity_total_api_keys":{"team-1":5}}} +{"results":[{"date":"2026-06-01","metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"breakdown":{"mcp_servers":{},"models":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"model_groups":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"providers":{"provider-a":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"endpoints":{"/v1/chat/completions":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"api_keys":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}},"entities":{"team-1":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{"team_alias":"Usage Team"},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}}},"guardrails":{}}}],"metadata":{"total_spend":1273.0,"total_flat_cost":0.0,"total_ptu_hours":0.0,"total_prompt_tokens":1014,"total_completion_tokens":10,"total_tokens":1024,"total_api_requests":5,"total_successful_requests":5,"total_failed_requests":0,"total_cache_read_input_tokens":6,"total_cache_creation_input_tokens":0,"total_compression_saved_tokens":0,"total_compression_savings_spend":0.0,"total_prompt_caching_savings_spend":0.0,"total_gateway_injected_caching_savings_spend":0.0,"total_autorouter_savings_spend":0.0,"total_response_time_ms":0,"total_timed_requests":0,"page":1,"total_pages":1,"has_more":false,"api_key_limit":100,"total_api_keys":5,"entity_total_api_keys":{"team-1":5}}} diff --git a/tests/integration/spend/fixtures/daily_activity_user.json b/tests/integration/spend/fixtures/daily_activity_user.json index c4302ee7045..ae05e661071 100644 --- a/tests/integration/spend/fixtures/daily_activity_user.json +++ b/tests/integration/spend/fixtures/daily_activity_user.json @@ -1 +1 @@ -{"results":[{"date":"2026-06-01","metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"breakdown":{"mcp_servers":{},"models":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"model_groups":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"providers":{"provider-a":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"endpoints":{"/v1/chat/completions":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"api_keys":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}},"entities":{"user-1":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}}}}}],"metadata":{"total_spend":1273.0,"total_flat_cost":0.0,"total_ptu_hours":0.0,"total_prompt_tokens":1014,"total_completion_tokens":10,"total_tokens":1024,"total_api_requests":5,"total_successful_requests":5,"total_failed_requests":0,"total_cache_read_input_tokens":6,"total_cache_creation_input_tokens":0,"total_compression_saved_tokens":0,"total_compression_savings_spend":0.0,"total_prompt_caching_savings_spend":0.0,"total_gateway_injected_caching_savings_spend":0.0,"total_autorouter_savings_spend":0.0,"total_response_time_ms":0,"total_timed_requests":0,"page":1,"total_pages":1,"has_more":false,"api_key_limit":100,"total_api_keys":5,"entity_total_api_keys":{"user-1":5}}} +{"results":[{"date":"2026-06-01","metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"breakdown":{"mcp_servers":{},"models":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"model_groups":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"providers":{"provider-a":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"endpoints":{"/v1/chat/completions":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"api_keys":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}},"entities":{"user-1":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}}},"guardrails":{}}}],"metadata":{"total_spend":1273.0,"total_flat_cost":0.0,"total_ptu_hours":0.0,"total_prompt_tokens":1014,"total_completion_tokens":10,"total_tokens":1024,"total_api_requests":5,"total_successful_requests":5,"total_failed_requests":0,"total_cache_read_input_tokens":6,"total_cache_creation_input_tokens":0,"total_compression_saved_tokens":0,"total_compression_savings_spend":0.0,"total_prompt_caching_savings_spend":0.0,"total_gateway_injected_caching_savings_spend":0.0,"total_autorouter_savings_spend":0.0,"total_response_time_ms":0,"total_timed_requests":0,"page":1,"total_pages":1,"has_more":false,"api_key_limit":100,"total_api_keys":5,"entity_total_api_keys":{"user-1":5}}} diff --git a/tests/integration/spend/golden/daily_activity_team_aggregated.json b/tests/integration/spend/golden/daily_activity_team_aggregated.json index 2956adda17a..91a5d33391b 100644 --- a/tests/integration/spend/golden/daily_activity_team_aggregated.json +++ b/tests/integration/spend/golden/daily_activity_team_aggregated.json @@ -26,7 +26,7 @@ }, "results": [ { - "breakdown": { + "breakdown": {"guardrails": {}, "api_keys": { "key-a": { "metadata": { diff --git a/tests/integration/spend/golden/daily_activity_team_paginated.json b/tests/integration/spend/golden/daily_activity_team_paginated.json index ca7239f8fa9..4d91b0da209 100644 --- a/tests/integration/spend/golden/daily_activity_team_paginated.json +++ b/tests/integration/spend/golden/daily_activity_team_paginated.json @@ -26,7 +26,7 @@ }, "results": [ { - "breakdown": { + "breakdown": {"guardrails": {}, "api_keys": { "key-a": { "metadata": { diff --git a/tests/integration/spend/golden/daily_activity_user_aggregated.json b/tests/integration/spend/golden/daily_activity_user_aggregated.json index 261ac22b2cb..185d333965a 100644 --- a/tests/integration/spend/golden/daily_activity_user_aggregated.json +++ b/tests/integration/spend/golden/daily_activity_user_aggregated.json @@ -26,7 +26,7 @@ }, "results": [ { - "breakdown": { + "breakdown": {"guardrails": {}, "api_keys": { "key-a": { "metadata": { diff --git a/tests/integration/spend/golden/daily_activity_user_paginated.json b/tests/integration/spend/golden/daily_activity_user_paginated.json index f1fb7ba50a4..00389289ebe 100644 --- a/tests/integration/spend/golden/daily_activity_user_paginated.json +++ b/tests/integration/spend/golden/daily_activity_user_paginated.json @@ -26,7 +26,7 @@ }, "results": [ { - "breakdown": { + "breakdown": {"guardrails": {}, "api_keys": { "key-a": { "metadata": { diff --git a/tests/integration/spend/test_daily_activity_aggregated_breakdowns.py b/tests/integration/spend/test_daily_activity_aggregated_breakdowns.py index 33baf9d0e2e..1c315a6e694 100644 --- a/tests/integration/spend/test_daily_activity_aggregated_breakdowns.py +++ b/tests/integration/spend/test_daily_activity_aggregated_breakdowns.py @@ -243,3 +243,183 @@ async def test_get_daily_activity_aggregated_model_group_rollups_fall_back_to_mo assert object_value(object_value(models["gpt-5"])["metrics"])["spend"] == 10.0 finally: _clean(day) + + +def _seed_guardrail_units(day: str, rows: Sequence[tuple[str, str, str, float | None, int, int]]) -> None: + for guardrail_id, api_key, usage_unit, cost, units, untracked_units in rows: + write_rows( + 'INSERT INTO "LiteLLM_DailyGuardrailUsageUnits" (guardrail_id, date, team_id, api_key, usage_unit,' + " units, cost, untracked_units, updated_at) VALUES (%s, %s, '', %s, %s, %s, %s, %s, now())", + ( + guardrail_id, + day, + api_key, + usage_unit, + str(units), + None if cost is None else str(cost), + str(untracked_units), + ), + ) + + +def _guardrail_spend(breakdown: dict[str, JsonValue]) -> dict[str, tuple[float, dict[str, float]]]: + return { + guardrail_id: ( + float(object_value(object_value(entry)["metrics"])["spend"]), + { + api_key: float(object_value(object_value(key_entry)["metrics"])["spend"]) + for api_key, key_entry in object_value(object_value(entry)["api_key_breakdown"]).items() + }, + ) + for guardrail_id, entry in object_value(breakdown["guardrails"]).items() + } + + +def _day_breakdown(gateway: Gateway, day: str, **params: str) -> dict[str, JsonValue]: + results: Final = _RESULTS.validate_python(_activity(gateway, day, **params)["results"]) + assert len(results) == 1 + return object_value(object_value(results[0])["breakdown"]) + + +@pytest.mark.asyncio +async def test_get_daily_activity_aggregated_breaks_down_guardrail_spend(gateway: Gateway) -> None: + day: Final = _unique_day() + _seed( + day, + [ + (_row_id(), "user-a", day, "key-a", "gpt-5", "", "bedrock", None, "/v1/chat/completions", 10, 1.0, 1, 1), + (_row_id(), "user-b", day, "key-b", "gpt-5", "", "bedrock", None, "/v1/chat/completions", 10, 2.0, 1, 1), + ], + ) + _seed_guardrail_units( + day, + [ + ("pii-filter", "key-a", "contentPolicyUnits", 0.10, 5, 2), + ("pii-filter", "key-a", "sensitiveInformationPolicyUnits", 0.05, 1, 0), + ("pii-filter", "key-b", "contentPolicyUnits", 0.20, 1, 0), + ("pii-filter", "key-b", "legacyUnits", None, 7, 0), + ("topic-filter", "key-a", "topicPolicyUnits", 0.01, 1, 0), + ("topic-filter", "key-b", "topicPolicyUnits", None, 1, 0), + ("pii-filter", "orphan-key", "contentPolicyUnits", 9.0, 1, 0), + ], + ) + try: + everyone: Final = _day_breakdown(gateway, day) + assert _guardrail_spend(everyone) == { + "pii-filter": (pytest.approx(0.35), {"key-a": pytest.approx(0.15), "key-b": pytest.approx(0.20)}), + "topic-filter": (pytest.approx(0.01), {"key-a": pytest.approx(0.01), "key-b": 0.0}), + } + api_keys: Final = object_value(everyone["api_keys"]) + pii_filter: Final = object_value(object_value(everyone["guardrails"])["pii-filter"]) + assert object_value(pii_filter["metrics"]) == {"spend": pytest.approx(0.35), "untracked_units": 9} + pii_api_keys: Final = object_value(pii_filter["api_key_breakdown"]) + assert object_value(object_value(pii_api_keys["key-a"])["metrics"]) == { + "spend": pytest.approx(0.15), + "untracked_units": 2, + } + assert object_value(object_value(pii_api_keys["key-b"])["metrics"]) == { + "spend": pytest.approx(0.20), + "untracked_units": 7, + } + + assert object_value(everyone["guardrails"])["topic-filter"] == { + "metrics": {"spend": pytest.approx(0.01), "untracked_units": 1}, + "metadata": {"guardrail_name": "topic-filter"}, + "api_key_breakdown": { + "key-a": { + "metrics": {"spend": pytest.approx(0.01), "untracked_units": 0}, + "metadata": object_value(api_keys["key-a"])["metadata"], + }, + "key-b": { + "metrics": {"spend": 0.0, "untracked_units": 1}, + "metadata": object_value(api_keys["key-b"])["metadata"], + }, + }, + } + assert object_value(object_value(object_value(everyone["models"])["gpt-5"])["metrics"])["spend"] == 3.0 + + assert _guardrail_spend(_day_breakdown(gateway, day, user_id="user-a")) == { + "pii-filter": (pytest.approx(0.15), {"key-a": pytest.approx(0.15)}), + "topic-filter": (pytest.approx(0.01), {"key-a": pytest.approx(0.01)}), + } + assert _guardrail_spend(_day_breakdown(gateway, day, api_key_limit="1")) == { + "pii-filter": (pytest.approx(0.35), {"key-b": pytest.approx(0.20)}), + "topic-filter": (pytest.approx(0.01), {"key-b": 0.0}), + } + limited_pii_filter: Final = object_value( + object_value(_day_breakdown(gateway, day, api_key_limit="1")["guardrails"])["pii-filter"] + ) + assert object_value(limited_pii_filter["metrics"])["untracked_units"] == 9 + assert set(object_value(limited_pii_filter["api_key_breakdown"])) == {"key-b"} + assert _guardrail_spend(_day_breakdown(gateway, day, model="gpt-5")) == {} + assert _guardrail_spend(_day_breakdown(gateway, day, model="")) == _guardrail_spend(everyone) + finally: + _clean(day) + write_rows('DELETE FROM "LiteLLM_DailyGuardrailUsageUnits" WHERE date = %s', (day,)) + + +def _entity_breakdown(gateway: Gateway, url: str, day: str, **params: str) -> dict[str, JsonValue]: + response: Final = gateway.request("GET", url, params={"start_date": day, "end_date": day, **params}) + assert response.status_code == 200, response.text + results: Final = _RESULTS.validate_python(object_value(response.json())["results"]) + assert len(results) == 1 + return object_value(object_value(results[0])["breakdown"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("url", "table", "entity_column", "filter_param"), + [ + ("/team/daily/activity/aggregated", "LiteLLM_DailyTeamSpend", "team_id", "team_ids"), + ( + "/organization/daily/activity/aggregated", + "LiteLLM_DailyOrganizationSpend", + "organization_id", + "organization_ids", + ), + ], +) +async def test_team_and_organization_activity_break_down_guardrail_spend_by_name( + gateway: Gateway, url: str, table: str, entity_column: str, filter_param: str +) -> None: + day: Final = _unique_day() + db_guardrail_id: Final = str(uuid.uuid4()) + db_guardrail_name: Final = f"pii-filter-{db_guardrail_id[:8]}" + for entity_id, api_key, spend in (("entity-a", "key-a", 1.0), ("entity-b", "key-b", 2.0)): + write_rows( + f'INSERT INTO "{table}" (id, {entity_column}, date, api_key, model, spend, api_requests,' + " successful_requests, updated_at) VALUES (%s, %s, %s, %s, 'gpt-5', %s, 1, 1, now())", + (_row_id(), entity_id, day, api_key, str(spend)), + ) + write_rows( + 'INSERT INTO "LiteLLM_GuardrailsTable" (guardrail_id, guardrail_name, litellm_params, updated_at)' + " VALUES (%s, %s, '{}', now())", + (db_guardrail_id, db_guardrail_name), + ) + _seed_guardrail_units( + day, + [ + (db_guardrail_id, "key-a", "contentPolicyUnits", 0.10, 1, 0), + (db_guardrail_id, "key-b", "contentPolicyUnits", 0.20, 1, 0), + ("config-topic-filter", "key-a", "topicPolicyUnits", 0.05, 1, 0), + (db_guardrail_id, "orphan-key", "contentPolicyUnits", 9.0, 1, 0), + ], + ) + try: + everyone: Final = _entity_breakdown(gateway, url, day) + assert _guardrail_spend(everyone) == { + db_guardrail_id: (pytest.approx(0.30), {"key-a": pytest.approx(0.10), "key-b": pytest.approx(0.20)}), + "config-topic-filter": (pytest.approx(0.05), {"key-a": pytest.approx(0.05)}), + } + guardrails: Final = object_value(everyone["guardrails"]) + assert object_value(guardrails[db_guardrail_id])["metadata"] == {"guardrail_name": db_guardrail_name} + assert object_value(guardrails["config-topic-filter"])["metadata"] == {"guardrail_name": "config-topic-filter"} + + assert _guardrail_spend(_entity_breakdown(gateway, url, day, **{filter_param: "entity-a"})) == { + db_guardrail_id: (pytest.approx(0.10), {"key-a": pytest.approx(0.10)}), + "config-topic-filter": (pytest.approx(0.05), {"key-a": pytest.approx(0.05)}), + } + finally: + write_rows(f'DELETE FROM "{table}" WHERE date = %s', (day,)) + write_rows('DELETE FROM "LiteLLM_DailyGuardrailUsageUnits" WHERE date = %s', (day,)) + write_rows('DELETE FROM "LiteLLM_GuardrailsTable" WHERE guardrail_id = %s', (db_guardrail_id,)) diff --git a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py index 8fbead8c01c..2a273034a97 100644 --- a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py +++ b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py @@ -248,6 +248,7 @@ class _FakeRepository: def __init__(self, rows: tuple[_Activity, ...]) -> None: self._rows: Final = rows self.aggregated = AsyncMock(side_effect=self._aggregated) + self.guardrail_spend = AsyncMock(return_value=()) self.key_page = AsyncMock(side_effect=self._key_page) self.key_page_call: tuple[DailyActivityScope, int, int] | None = None self.search_keys = AsyncMock(side_effect=self._search_keys) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 70406105193..72cc8cdc94e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27721,6 +27721,10 @@ export interface components { entities?: { [key: string]: components["schemas"]["MetricWithMetadata"]; }; + /** Guardrails */ + guardrails?: { + [key: string]: components["schemas"]["GuardrailMetric"]; + }; /** Mcp Servers */ mcp_servers?: { [key: string]: components["schemas"]["MetricWithMetadata"]; @@ -33236,6 +33240,38 @@ export interface components { /** Updated At */ updated_at?: string | null; }; + /** GuardrailKeyMetric */ + GuardrailKeyMetric: { + metadata: components["schemas"]["KeyMetadata"]; + metrics: components["schemas"]["GuardrailSpendMetrics"]; + }; + /** GuardrailMetadata */ + GuardrailMetadata: { + /** Guardrail Name */ + guardrail_name: string; + }; + /** GuardrailMetric */ + GuardrailMetric: { + /** Api Key Breakdown */ + api_key_breakdown?: { + [key: string]: components["schemas"]["GuardrailKeyMetric"]; + }; + metadata: components["schemas"]["GuardrailMetadata"]; + metrics: components["schemas"]["GuardrailSpendMetrics"]; + }; + /** GuardrailSpendMetrics */ + GuardrailSpendMetrics: { + /** + * Spend + * @default 0 + */ + spend: number; + /** + * Untracked Units + * @default 0 + */ + untracked_units: number; + }; /** GuardrailSubmissionItem */ GuardrailSubmissionItem: { /** Created At */ From cf1006a3687de717f247e846cefedf823f2d8203 Mon Sep 17 00:00:00 2001 From: mrinal-berri Date: Wed, 28 Oct 2026 15:35:33 -0700 Subject: [PATCH 09/27] feat(ui): search and multi-select tools in the MCP toolset editor (#48601) * feat(ui): add per-server MCP tool catalog query hook One query per server keyed by server_id, so the toolset editor can load tools for many servers in parallel and a failing server surfaces as an error on its own row instead of an empty list. (LIT-10012) * feat(ui): add toolset tool search and bulk selection helpers Pure functions for the toolset editor: filter a server's tools by a search query, select/deselect many tools without duplicates or dropping other selections, and compute tri-state selection. (LIT-10012) * feat(ui): search tools and bulk select in the toolset editor The toolset editor search now matches tool names and descriptions as well as server names. Typing fetches every server's tools in parallel through the per-server query hook, auto-expands matching servers and hides servers that loaded with no match. Tool rows become checkboxes, each server header gets a tri-state select-all over its visible tools, and a bar offers select/deselect all matching tools. Bulk actions only touch visible tools, so selections on other servers are kept on save. (LIT-10012) * feat(ui): clear all and flag unconfirmed tools in the toolset panel Adds a Clear all action to the Your Toolset panel and labels selected tools whose server is gone, unreachable, or no longer lists the tool. Those tools stay selected and are still saved, so an outage never silently drops part of a toolset. (LIT-10012) * test(ui): cover toolset editor tool search and bulk selection Integration tests that drive the real toolset modal against stubbed networking and assert the saved tools payload: cross-server search by tool name and description, select/deselect all matching, server select-all, single-row toggles, Clear all, and that tools on unreachable or deleted servers survive a save. (LIT-10012) * fix(ui): pointer cursor on the toolset server select-all checkbox Base UI renders the checkbox as a span, which the global button cursor rule does not cover, so the server header select-all showed the default arrow. Use a pointer, and not-allowed while it is disabled. (LIT-10012) * fix(ui): stop toolset editor background fan-out and keep tools on failed refetch The toolset modal stays mounted when closed, so a finished search kept every server's tool query enabled and refocusing the window refetched all of them once stale. Fetch only while the editor is open, require both the live and debounced search before fanning out, and disable refetch on window focus for these queries. A failed refetch now keeps the tools that already loaded instead of flagging the server unreachable, and tools without a name are dropped. Trim comments that restated the code. (LIT-10012) * fix(ui): clear toolset search on close and only keep non-empty tool lists on failed refetch Reopening the editor after a search rendered once with the old query, which fetched every stale server; clear the search whenever the editor closes. A failed refetch now keeps the previous list only when it has tools, so a server that loaded none and then fails reports the error. (LIT-10012) * fix(ui): reset toolset editor state on close and avoid mutating selection sets Closing the editor now also clears expanded servers, so reopening it cannot refetch them for one render. Build the selection and expanded-server sets without in-place mutation, drop a comment that restated the code, and use screen queries in the new tests to stay within the prefer-screen-queries lint budget. (LIT-10012) --- ui/litellm-dashboard/eslint-suppressions.json | 3 - .../mcpServers/useMCPServerTools.test.ts | 118 ++++ .../hooks/mcpServers/useMCPServerTools.ts | 70 +++ .../MCPToolsetsTab.integration.test.tsx | 219 +++++++- .../_components/MCPToolsetsTab.tsx | 503 ++++++++++++------ .../_components/toolsetSelection.test.ts | 90 ++++ .../_components/toolsetSelection.ts | 63 +++ 7 files changed, 890 insertions(+), 176 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerTools.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerTools.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/toolsetSelection.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/toolsetSelection.ts diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index e67c1105d86..9fce93782b2 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -498,9 +498,6 @@ } }, "src/app/(dashboard)/mcp-servers/_components/MCPToolsetsTab.tsx": { - "no-nested-ternary": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 1 } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerTools.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerTools.test.ts new file mode 100644 index 00000000000..3ef8e88b2c9 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerTools.test.ts @@ -0,0 +1,118 @@ +/* @vitest-environment jsdom */ +import React from "react"; +import { act, renderHook, waitFor } from "@testing-library/react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { mcpServerToolKeys, useMCPServerTools } from "./useMCPServerTools"; +import * as networking from "@/components/networking"; + +vi.mock("@/components/networking", () => ({ + listMCPTools: vi.fn(), +})); + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: vi.fn(() => ({ accessToken: "test-token-123" })), +})); + +const wrapper = ({ children }: { children: React.ReactNode }) => { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); + return React.createElement(QueryClientProvider, { client: queryClient }, children); +}; + +const toolsFor = (serverId: string) => ({ + tools: [{ name: `${serverId}_tool`, description: `from ${serverId}` }], + error: null, +}); + +describe("useMCPServerTools", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("fetches only the enabled servers and leaves the rest idle", async () => { + vi.mocked(networking.listMCPTools).mockImplementation(async (_token, serverId) => toolsFor(serverId)); + + const { result } = renderHook(() => useMCPServerTools(["a", "b"], new Set(["a"])), { wrapper }); + + await waitFor(() => expect(result.current.get("a")?.tools).toEqual([{ name: "a_tool", description: "from a" }])); + expect(result.current.get("b")).toEqual({ tools: [], status: "idle" }); + expect(networking.listMCPTools).toHaveBeenCalledTimes(1); + expect(networking.listMCPTools).toHaveBeenCalledWith("test-token-123", "a"); + }); + + it("isolates a failing server from the others", async () => { + vi.mocked(networking.listMCPTools).mockImplementation(async (_token, serverId) => + serverId === "bad" ? { tools: [], error: "http_500", message: "upstream down" } : toolsFor(serverId), + ); + + const { result } = renderHook(() => useMCPServerTools(["good", "bad"], new Set(["good", "bad"])), { wrapper }); + + await waitFor(() => expect(result.current.get("bad")).toEqual({ tools: [], status: "error" })); + expect(result.current.get("good")).toEqual({ + tools: [{ name: "good_tool", description: "from good" }], + status: "success", + }); + }); + + it("reports a server that is being fetched as loading", () => { + vi.mocked(networking.listMCPTools).mockReturnValue(new Promise(() => {})); + + const { result } = renderHook(() => useMCPServerTools(["a"], new Set(["a"])), { wrapper }); + + expect(result.current.get("a")).toEqual({ tools: [], status: "loading" }); + }); + + it("normalizes a bare array response, tools without descriptions, and drops tools without a name", async () => { + vi.mocked(networking.listMCPTools).mockResolvedValue([ + { tool_name: "legacy" }, + "plain", + { description: "nameless" }, + ]); + + const { result } = renderHook(() => useMCPServerTools(["a"], new Set(["a"])), { wrapper }); + + await waitFor(() => + expect(result.current.get("a")?.tools).toEqual([ + { name: "legacy", description: "" }, + { name: "plain", description: "" }, + ]), + ); + }); + + it("reports an error when a refetch fails for a server that had loaded no tools", async () => { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + const sharedWrapper = ({ children }: { children: React.ReactNode }) => + React.createElement(QueryClientProvider, { client: queryClient }, children); + vi.mocked(networking.listMCPTools) + .mockResolvedValueOnce({ tools: [], error: null }) + .mockResolvedValueOnce({ tools: [], error: "http_502", message: "upstream down" }); + + const { result } = renderHook(() => useMCPServerTools(["a"], new Set(["a"])), { wrapper: sharedWrapper }); + await waitFor(() => expect(result.current.get("a")?.status).toBe("success")); + + await act(() => queryClient.refetchQueries({ queryKey: mcpServerToolKeys.detail("a") })); + + await waitFor(() => expect(result.current.get("a")).toEqual({ tools: [], status: "error" })); + }); + + it("keeps showing the loaded tools when a later refetch fails", async () => { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + const sharedWrapper = ({ children }: { children: React.ReactNode }) => + React.createElement(QueryClientProvider, { client: queryClient }, children); + vi.mocked(networking.listMCPTools) + .mockResolvedValueOnce(toolsFor("a")) + .mockResolvedValueOnce({ tools: [], error: "http_502", message: "upstream down" }); + + const { result } = renderHook(() => useMCPServerTools(["a"], new Set(["a"])), { wrapper: sharedWrapper }); + await waitFor(() => expect(result.current.get("a")?.status).toBe("success")); + + await act(() => queryClient.refetchQueries({ queryKey: mcpServerToolKeys.detail("a") })); + + await waitFor(() => expect(queryClient.getQueryState(mcpServerToolKeys.detail("a"))?.status).toBe("error")); + expect(networking.listMCPTools).toHaveBeenCalledTimes(2); + expect(result.current.get("a")).toEqual({ + tools: [{ name: "a_tool", description: "from a" }], + status: "success", + }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerTools.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerTools.ts new file mode 100644 index 00000000000..bc4a6696a6e --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerTools.ts @@ -0,0 +1,70 @@ +import { useQueries, type QueryObserverResult } from "@tanstack/react-query"; +import { createQueryKeys } from "../common/queryKeysFactory"; +import { listMCPTools } from "@/components/networking"; +import useAuthorized from "../useAuthorized"; + +export const mcpServerToolKeys = createQueryKeys("mcpServerTools"); + +export interface MCPServerToolEntry { + name: string; + description: string; +} + +export type MCPServerToolsStatus = "idle" | "loading" | "error" | "success"; + +export interface MCPServerToolsState { + tools: MCPServerToolEntry[]; + status: MCPServerToolsStatus; +} + +type RawTool = string | { name?: string; tool_name?: string; description?: string }; + +const toEntry = (tool: RawTool): MCPServerToolEntry => + typeof tool === "string" + ? { name: tool, description: "" } + : { name: tool.name ?? tool.tool_name ?? "", description: tool.description ?? "" }; + +// listMCPTools never throws: it reports failures as `{ tools: [], error }`. +// Throw here so a broken server surfaces as a query error instead of an empty tool list. +export const fetchServerTools = async (accessToken: string, serverId: string): Promise => { + const result = await listMCPTools(accessToken, serverId); + if (!Array.isArray(result) && result?.error) { + throw new Error(result.message || result.error); + } + const toolList: RawTool[] = Array.isArray(result) ? result : result?.tools ?? []; + return toolList.map(toEntry).filter((tool) => tool.name !== ""); +}; + +const toStatus = ({ + status, + fetchStatus, + data, +}: Pick, "status" | "fetchStatus" | "data">): MCPServerToolsStatus => { + if (status === "error" && data !== undefined && data.length > 0) return "success"; + if (status !== "pending") return status; + return fetchStatus === "idle" ? "idle" : "loading"; +}; + +export const useMCPServerTools = ( + serverIds: readonly string[], + enabledServerIds: ReadonlySet, +): ReadonlyMap => { + const { accessToken } = useAuthorized(); + return useQueries({ + queries: serverIds.map((serverId) => ({ + queryKey: mcpServerToolKeys.detail(serverId), + queryFn: () => fetchServerTools(accessToken!, serverId), + enabled: !!accessToken && enabledServerIds.has(serverId), + staleTime: 5 * 60 * 1000, + retry: false, + refetchOnWindowFocus: false, + })), + combine: (results) => + new Map( + results.map((result, idx): [string, MCPServerToolsState] => [ + serverIds[idx], + { tools: result.data ?? [], status: toStatus(result) }, + ]), + ), + }); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPToolsetsTab.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPToolsetsTab.integration.test.tsx index 81462c6bf9b..ba9f2bfa839 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPToolsetsTab.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPToolsetsTab.integration.test.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import { act, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event"; import { describe, it, expect, vi, beforeEach } from "vitest"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; @@ -7,6 +7,7 @@ import { MCPToolsetsTab } from "./MCPToolsetsTab"; import * as networking from "@/components/networking"; import { useMCPToolsets } from "@/app/(dashboard)/hooks/mcpServers/useMCPToolsets"; import { useMCPServers } from "@/app/(dashboard)/hooks/mcpServers/useMCPServers"; +import { mcpServerToolKeys } from "@/app/(dashboard)/hooks/mcpServers/useMCPServerTools"; import { MCPToolset } from "@/components/mcp_tools/types"; vi.mock("@/components/networking", () => ({ @@ -20,6 +21,7 @@ vi.mock("@/components/networking", () => ({ vi.mock("@/app/(dashboard)/hooks/mcpServers/useMCPToolsets", () => ({ useMCPToolsets: vi.fn() })); vi.mock("@/app/(dashboard)/hooks/mcpServers/useMCPServers", () => ({ useMCPServers: vi.fn() })); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => ({ accessToken: "sk-test" }) })); const setup = () => userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); @@ -197,6 +199,221 @@ describe("MCPToolsetsTab create/edit toolset form", () => { }); }); +describe("MCPToolsetsTab tool search and bulk selection", () => { + const servers = [ + { server_id: "gh", alias: "github", server_name: "GitHub" }, + { server_id: "lin", alias: "linear", server_name: "Linear" }, + { server_id: "broken", alias: "flaky", server_name: "Flaky" }, + ]; + const catalogs: Record = { + gh: [ + { name: "create_issue", description: "Open a GitHub issue" }, + { name: "list_pulls", description: "List pull requests" }, + ], + lin: [ + { name: "create_ticket", description: "Create a Linear issue" }, + { name: "list_projects", description: "List projects" }, + ], + }; + + const renderWithServers = (toolsets: MCPToolset[] = []) => { + vi.mocked(useMCPToolsets).mockReturnValue({ + data: toolsets, + isLoading: false, + } as unknown as ReturnType); + vi.mocked(useMCPServers).mockReturnValue({ data: servers, isSuccess: true } as unknown as ReturnType< + typeof useMCPServers + >); + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); + render( + + + , + ); + return queryClient; + }; + + const editToolset = (tools: MCPToolset["tools"]) => + ({ toolset_id: "ts-1", toolset_name: "triage", description: "", tools }) as MCPToolset; + + const expectedUpdate = (tools: MCPToolset["tools"]) => ({ + toolset_id: "ts-1", + toolset_name: "triage", + description: "", + tools, + }); + + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(networking.listMCPTools).mockImplementation(async (_token, serverId) => + serverId in catalogs + ? { tools: catalogs[serverId], error: null } + : { tools: [], error: "http_502", message: "upstream down" }, + ); + vi.mocked(networking.createMCPToolset).mockResolvedValue( + {} as Awaited>, + ); + vi.mocked(networking.updateMCPToolset).mockResolvedValue( + {} as Awaited>, + ); + }); + + it("searches tool names and descriptions across collapsed servers and selects every match", async () => { + const user = setup(); + renderWithServers(); + + await openCreate(user); + fireEvent.change(screen.getByPlaceholderText("e.g. github-linear-tools"), { target: { value: "issues" } }); + await user.type(screen.getByRole("textbox", { name: "Search tools or servers" }), "issue"); + + expect(await screen.findByRole("checkbox", { name: /^create_ticket/ })).toBeInTheDocument(); + expect(screen.getByRole("checkbox", { name: /^create_issue/ })).toBeInTheDocument(); + expect(screen.queryByRole("checkbox", { name: /^list_pulls/ })).not.toBeInTheDocument(); + expect(await screen.findByText("2 matching tools")).toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Select all matching" })); + await user.click(screen.getByRole("button", { name: "Create Toolset" })); + + await waitFor(() => { + expect(networking.createMCPToolset).toHaveBeenCalledWith("sk-test", { + toolset_name: "issues", + description: "", + tools: [ + { server_id: "gh", tool_name: "create_issue" }, + { server_id: "lin", tool_name: "create_ticket" }, + ], + }); + }); + }); + + it("deselecting all matches keeps selected tools the search hides", async () => { + const user = setup(); + renderWithServers([ + editToolset([ + { server_id: "gh", tool_name: "list_pulls" }, + { server_id: "gh", tool_name: "create_issue" }, + { server_id: "lin", tool_name: "create_ticket" }, + ]), + ]); + + await openEditFor(user); + await user.type(screen.getByRole("textbox", { name: "Search tools or servers" }), "create"); + expect(await screen.findByRole("checkbox", { name: /^create_ticket/ })).toBeChecked(); + + await user.click(screen.getByRole("button", { name: "Deselect all matching" })); + await user.click(screen.getByRole("button", { name: "Save Changes" })); + + await waitFor(() => { + expect(networking.updateMCPToolset).toHaveBeenCalledWith( + "sk-test", + expectedUpdate([{ server_id: "gh", tool_name: "list_pulls" }]), + ); + }); + }); + + it("selects all of one server's tools from its header and toggles single rows, fetching only that server", async () => { + const user = setup(); + renderWithServers(); + + await openCreate(user); + fireEvent.change(screen.getByPlaceholderText("e.g. github-linear-tools"), { target: { value: "gh-only" } }); + await user.click(screen.getByRole("button", { name: /^github/ })); + expect(await screen.findByRole("checkbox", { name: /^list_pulls/ })).toBeInTheDocument(); + + await user.click(screen.getByRole("checkbox", { name: "Select all tools on github" })); + expect(screen.getByRole("checkbox", { name: /^list_pulls/ })).toBeChecked(); + // Clicking anywhere on a tool row toggles it exactly once. + await user.click(screen.getByText("List pull requests")); + expect(screen.getByRole("checkbox", { name: /^list_pulls/ })).not.toBeChecked(); + await user.click(screen.getByRole("button", { name: "Create Toolset" })); + + await waitFor(() => { + expect(networking.createMCPToolset).toHaveBeenCalledWith("sk-test", { + toolset_name: "gh-only", + description: "", + tools: [{ server_id: "gh", tool_name: "create_issue" }], + }); + }); + expect(networking.listMCPTools).toHaveBeenCalledTimes(1); + }); + + it("keeps and flags tools whose server is unreachable or gone when saving", async () => { + const user = setup(); + const tools = [ + { server_id: "broken", tool_name: "flaky_tool" }, + { server_id: "deleted", tool_name: "old_tool" }, + { server_id: "gh", tool_name: "create_issue" }, + ]; + renderWithServers([editToolset(tools)]); + + await openEditFor(user); + await user.type(screen.getByRole("textbox", { name: "Search tools or servers" }), "create"); + + expect(await screen.findByText("Couldn't load tools for this server.")).toBeInTheDocument(); + expect(await screen.findByText("server unreachable")).toBeInTheDocument(); + expect(screen.getByText("server not found")).toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Save Changes" })); + + await waitFor(() => { + expect(networking.updateMCPToolset).toHaveBeenCalledWith("sk-test", expectedUpdate(tools)); + }); + }); + + it("stops fetching server tools once the editor is closed after a search", async () => { + const user = setup(); + const queryClient = renderWithServers(); + + await openCreate(user); + await user.type(screen.getByRole("textbox", { name: "Search tools or servers" }), "issue"); + expect(await screen.findByRole("checkbox", { name: /^create_ticket/ })).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Cancel" })); + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + const callsWhileOpen = vi.mocked(networking.listMCPTools).mock.calls.length; + + await act(() => queryClient.invalidateQueries({ queryKey: mcpServerToolKeys.all })); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(networking.listMCPTools).toHaveBeenCalledTimes(callsWhileOpen); + + await openCreate(user); + expect(screen.getByRole("textbox", { name: "Search tools or servers" })).toHaveValue(""); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(networking.listMCPTools).toHaveBeenCalledTimes(callsWhileOpen); + }); + + it("does not refetch a previously expanded server when the editor is reopened", async () => { + const user = setup(); + const queryClient = renderWithServers(); + + await openCreate(user); + await user.click(screen.getByRole("button", { name: /^github/ })); + expect(await screen.findByRole("checkbox", { name: /^list_pulls/ })).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Cancel" })); + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + await act(() => queryClient.invalidateQueries({ queryKey: mcpServerToolKeys.all })); + + await openCreate(user); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(networking.listMCPTools).toHaveBeenCalledTimes(1); + expect(screen.queryByRole("checkbox", { name: /^list_pulls/ })).not.toBeInTheDocument(); + }); + + it("clears the whole toolset with Clear all", async () => { + const user = setup(); + renderWithServers([editToolset([{ server_id: "gh", tool_name: "create_issue" }])]); + + await openEditFor(user); + await user.click(screen.getByRole("button", { name: "Clear all" })); + expect(screen.getByText("No tools added yet")).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Save Changes" })); + + await waitFor(() => { + expect(networking.updateMCPToolset).toHaveBeenCalledWith("sk-test", expectedUpdate([])); + }); + }); +}); + describe("MCPToolsetsTab usage guide", () => { it("shows the toolset URL on the gateway host", () => { renderTab(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPToolsetsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPToolsetsTab.tsx index ff0e8f7b6a9..dc1477b79d3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPToolsetsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPToolsetsTab.tsx @@ -1,28 +1,39 @@ -import React, { useState, useCallback } from "react"; +import React, { useState } from "react"; import { z } from "zod"; import { SortingState } from "@tanstack/react-table"; import { Inbox, Plus, X } from "lucide-react"; import { useMCPToolsets } from "@/app/(dashboard)/hooks/mcpServers/useMCPToolsets"; import { useMCPServers } from "@/app/(dashboard)/hooks/mcpServers/useMCPServers"; import { useQueryClient } from "@tanstack/react-query"; -import { DataTable } from "@/components/shared/DataTable"; +import { useDebouncedValue } from "@tanstack/react-pacer/debouncer"; import { - createMCPToolset, - updateMCPToolset, - deleteMCPToolset, - listMCPTools, - getGatewayBaseUrl, -} from "@/components/networking"; -import { MCPToolset, MCPToolsetTool } from "@/components/mcp_tools/types"; + useMCPServerTools, + type MCPServerToolEntry, + type MCPServerToolsState, +} from "@/app/(dashboard)/hooks/mcpServers/useMCPServerTools"; +import { DataTable } from "@/components/shared/DataTable"; +import { createMCPToolset, updateMCPToolset, deleteMCPToolset, getGatewayBaseUrl } from "@/components/networking"; +import { MCPServer, MCPToolset, MCPToolsetTool } from "@/components/mcp_tools/types"; import { FieldGroup } from "@/components/ui/field"; import { FormField } from "@/components/shared/form/FormField"; import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; import { Input } from "@/components/ui/input"; import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { useZodForm } from "@/lib/forms/useZodForm"; import { toast } from "@/lib/toast"; import { displayToolName, getMCPToolsetTableColumns } from "./MCPToolsetTableColumns"; +import { + deselectTools, + filterServerTools, + isToolSelected, + matchesServerName, + selectTools, + selectionState, + toggleTool, + type SelectionState, +} from "./toolsetSelection"; import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; interface MCPToolsetsTabProps { @@ -37,110 +48,191 @@ const toolsetSchema = z.object({ type ToolsetFormValues = z.infer; -interface MCPToolListProps { +const SEARCH_DEBOUNCE_WAIT_MS = 300; + +const IDLE_SERVER_TOOLS: MCPServerToolsState = { tools: [], status: "idle" }; + +const NO_SERVERS: ReadonlySet = new Set(); + +// The modal stays mounted while closed, so fetch only while it is open; otherwise a finished +// search keeps every server's query live and refetches all of them in the background. +const serversToFetch = ( + open: boolean, + searching: boolean, + serverIds: readonly string[], + expandedServerIds: ReadonlySet, +): ReadonlySet => { + if (!open) return NO_SERVERS; + return searching ? new Set(serverIds) : expandedServerIds; +}; + +const checkboxStateProps = (state: SelectionState) => ({ + checked: state === "all", + indeterminate: state === "some", +}); + +interface ServerView { + server: MCPServer; + serverName: string; + state: MCPServerToolsState; + visibleTools: readonly MCPServerToolEntry[]; +} + +const isPending = (state: MCPServerToolsState) => state.status === "idle" || state.status === "loading"; + +// While searching, a server stays listed until its tools load and turn out not to match. +const isHiddenBySearch = ( + search: string, + serverNames: readonly string[], + state: MCPServerToolsState, + visibleTools: readonly MCPServerToolEntry[], +) => { + if (!search.trim() || state.status !== "success") return false; + return visibleTools.length === 0 && !matchesServerName(search, serverNames); +}; + +const buildServerViews = ( + servers: readonly MCPServer[], + toolsByServer: ReadonlyMap, + search: string, +): ServerView[] => + servers.flatMap((server) => { + const state = toolsByServer.get(server.server_id) ?? IDLE_SERVER_TOOLS; + const names = [server.alias ?? "", server.server_name ?? ""]; + const visibleTools = filterServerTools(search, names, state.tools); + if (isHiddenBySearch(search, names, state, visibleTools)) return []; + return [{ server, serverName: server.alias || server.server_name || server.server_id, state, visibleTools }]; + }); + +const summarizeSearch = (serverViews: readonly ServerView[]) => ({ + matchingTools: serverViews.flatMap(({ server, visibleTools }) => + visibleTools.map((tool) => ({ server_id: server.server_id, tool_name: tool.name })), + ), + pendingServerCount: serverViews.filter(({ state }) => isPending(state)).length, +}); + +// Why a selected tool cannot be confirmed against its server. The tool stays selected and is +// still saved; the note only keeps it from looking lost. +const selectedToolIssue = ( + tool: MCPToolsetTool, + serverNameById: ReadonlyMap, + toolsByServer: ReadonlyMap, +): string | undefined => { + if (!serverNameById.has(tool.server_id)) return "server not found"; + const state = toolsByServer.get(tool.server_id); + if (state?.status === "error") return "server unreachable"; + if (state?.status === "success" && !state.tools.some((t) => t.name === tool.tool_name)) return "tool not found"; + return undefined; +}; + +interface ServerToolGroupProps { serverId: string; serverName: string; - accessToken: string | null; + state: MCPServerToolsState; + visibleTools: readonly MCPServerToolEntry[]; + expanded: boolean; + searchActive: boolean; selectedTools: MCPToolsetTool[]; - onToggle: (tool: MCPToolsetTool) => void; + onToggleExpanded: () => void; + onToggleTool: (tool: MCPToolsetTool) => void; + onSetToolsSelected: (tools: MCPToolsetTool[], selected: boolean) => void; } -interface ToolEntry { - name: string; - description?: string; -} +function ServerToolGroup({ + serverId, + serverName, + state, + visibleTools, + expanded, + searchActive, + selectedTools, + onToggleExpanded, + onToggleTool, + onSetToolsSelected, +}: ServerToolGroupProps) { + const selectedCount = selectedTools.filter((t) => t.server_id === serverId).length; + const visibleRefs = visibleTools.map((tool) => ({ server_id: serverId, tool_name: tool.name })); + const groupState = selectionState(selectedTools, visibleRefs); -function MCPToolList({ serverId, serverName, accessToken, selectedTools, onToggle }: MCPToolListProps) { - const [tools, setTools] = useState([]); - const [loading, setLoading] = useState(false); - const [expanded, setExpanded] = useState(false); - - const selectedSet = new Set(selectedTools.filter((t) => t.server_id === serverId).map((t) => t.tool_name)); - - const fetchTools = useCallback(async () => { - if (!accessToken || tools.length > 0) return; - setLoading(true); - try { - const result = await listMCPTools(accessToken, serverId); - const toolList = Array.isArray(result) ? result : result?.tools ?? []; - setTools(toolList.map((t: any) => ({ name: t.name ?? t.tool_name ?? t, description: t.description ?? "" }))); - } catch { - setTools([]); - } finally { - setLoading(false); + const renderBody = () => { + if (isPending(state)) { + return ( +
+ +
+ ); } - }, [accessToken, serverId, tools.length]); - - const handleToggle = () => { - if (!expanded) fetchTools(); - setExpanded(!expanded); + if (state.status === "error") { + return

Couldn't load tools for this server.

; + } + if (visibleTools.length === 0) { + return ( +

+ {searchActive ? "No matching tools on this server." : "No tools found for this server."} +

+ ); + } + return ( +
+ {visibleRefs.map((ref, idx) => { + const tool = visibleTools[idx]; + const selected = isToolSelected(selectedTools, ref); + return ( + + ); + })} +
+ ); }; return (
- - {expanded && ( -
- {loading ? ( -
- -
- ) : tools.length === 0 ? ( -

No tools found for this server.

- ) : ( -
- {tools.map((tool) => { - const selected = selectedSet.has(tool.name); - return ( - - ); - })} -
- )} -
- )} +
+ onSetToolsSelected(visibleRefs, checked)} + /> + +
+ {expanded &&
{renderBody()}
}
); } @@ -149,21 +241,29 @@ interface CreateToolsetModalProps { open: boolean; onClose: () => void; onSave: (name: string, description: string | undefined, tools: MCPToolsetTool[]) => Promise; - accessToken: string | null; initialToolset?: MCPToolset; } -function CreateToolsetModal({ open, onClose, onSave, accessToken, initialToolset }: CreateToolsetModalProps) { - const form = useZodForm(toolsetSchema, { - defaultValues: { - toolset_name: initialToolset?.toolset_name || "", - description: initialToolset?.description || "", - }, - }); +const toolsetFormDefaults = (toolset?: MCPToolset): ToolsetFormValues => ({ + toolset_name: toolset?.toolset_name || "", + description: toolset?.description || "", +}); + +function CreateToolsetModal({ open, onClose, onSave, initialToolset }: CreateToolsetModalProps) { + const form = useZodForm(toolsetSchema, { defaultValues: toolsetFormDefaults(initialToolset) }); const [selectedTools, setSelectedTools] = useState(initialToolset?.tools || []); const [saving, setSaving] = useState(false); - const [serverSearch, setServerSearch] = useState(""); - const { data: mcpServers = [] } = useMCPServers(); + const [search, setSearch] = useState(""); + const [debouncedSearch] = useDebouncedValue(search, { wait: SEARCH_DEBOUNCE_WAIT_MS }); + const [expandedServerIds, setExpandedServerIds] = useState>(() => new Set()); + const { data: mcpServers = [], isSuccess: serversLoaded } = useMCPServers(); + const serverIds = React.useMemo(() => mcpServers.map((s) => s.server_id), [mcpServers]); + const searching = search.trim() !== "" && debouncedSearch.trim() !== ""; + const fetchServerIds = React.useMemo( + () => serversToFetch(open, searching, serverIds, expandedServerIds), + [open, searching, serverIds, expandedServerIds], + ); + const toolsByServer = useMCPServerTools(serverIds, fetchServerIds); const serverPrefixById = React.useMemo( () => new Map(mcpServers.map((s) => [s.server_id, s.alias || s.server_name || s.server_id])), [mcpServers], @@ -171,41 +271,54 @@ function CreateToolsetModal({ open, onClose, onSave, accessToken, initialToolset React.useEffect(() => { if (open) { - form.reset({ - toolset_name: initialToolset?.toolset_name || "", - description: initialToolset?.description || "", - }); + form.reset(toolsetFormDefaults(initialToolset)); setSelectedTools(initialToolset?.tools || []); - setServerSearch(""); + setSearch(""); + setExpandedServerIds(new Set()); } }, [open, initialToolset, form]); const handleToggleTool = (tool: MCPToolsetTool) => { - setSelectedTools((prev) => { - const exists = prev.some((t) => t.server_id === tool.server_id && t.tool_name === tool.tool_name); - return exists - ? prev.filter((t) => !(t.server_id === tool.server_id && t.tool_name === tool.tool_name)) - : [...prev, tool]; - }); + setSelectedTools((prev) => toggleTool(prev, tool)); + }; + + const handleSetToolsSelected = (tools: MCPToolsetTool[], selected: boolean) => { + setSelectedTools((prev) => (selected ? selectTools(prev, tools) : deselectTools(prev, tools))); + }; + + const handleToggleExpanded = (serverId: string) => { + setExpandedServerIds((prev) => + prev.has(serverId) ? new Set([...prev].filter((id) => id !== serverId)) : new Set([...prev, serverId]), + ); + }; + + // The create modal stays mounted, so state left from the last session would fetch servers + // on the first render after reopening, before the open effect resets it. + const handleClose = () => { + setSearch(""); + setExpandedServerIds(new Set()); + onClose(); }; const handleSubmit = async (values: ToolsetFormValues) => { setSaving(true); try { await onSave(values.toolset_name, values.description, selectedTools); - onClose(); + handleClose(); } finally { setSaving(false); } }; - const filteredServers = mcpServers.filter((s) => { - const q = serverSearch.toLowerCase(); - return !q || (s.alias || "").toLowerCase().includes(q) || (s.server_name || "").toLowerCase().includes(q); - }); + const searchActive = search.trim() !== ""; + const serverViews = buildServerViews(mcpServers, toolsByServer, search); + const { matchingTools, pendingServerCount } = searchActive + ? summarizeSearch(serverViews) + : { matchingTools: [], pendingServerCount: 0 }; + const matchingState = selectionState(selectedTools, matchingTools); return ( - !open && onClose()}> + !open && handleClose()}> {initialToolset ? "Edit Toolset" : "New Toolset"} @@ -229,32 +342,65 @@ function CreateToolsetModal({ open, onClose, onSave, accessToken, initialToolset setServerSearch(e.target.value)} + placeholder="Search tools or servers..." + aria-label="Search tools or servers" + value={search} + onChange={(e) => setSearch(e.target.value)} /> - {serverSearch && ( + {search && ( - setServerSearch("")}> + setSearch("")}> )} + {searchActive && ( +
+

+ {matchingTools.length} matching {matchingTools.length === 1 ? "tool" : "tools"} + {pendingServerCount > 0 && + ` · loading ${pendingServerCount} ${pendingServerCount === 1 ? "server" : "servers"}…`} +

+
+ + +
+
+ )}
- {filteredServers.length === 0 ? ( + {serverViews.length === 0 ? (

- {mcpServers.length === 0 ? "No MCP servers configured" : "No servers match your search"} + {mcpServers.length === 0 ? "No MCP servers configured" : "No tools or servers match your search"}

) : ( - filteredServers.map((server) => ( - ( + handleToggleExpanded(server.server_id)} + onToggleTool={handleToggleTool} + onSetToolsSelected={handleSetToolsSelected} /> )) )} @@ -266,41 +412,60 @@ function CreateToolsetModal({ open, onClose, onSave, accessToken, initialToolset {/* Right panel: Your Toolset */}
-

- Your Toolset{" "} - ({selectedTools.length} tools) -

+
+

+ Your Toolset{" "} + ({selectedTools.length} tools) +

+ +
{selectedTools.length === 0 ? (

No tools added yet

) : ( - selectedTools.map((tool, idx) => ( -
- - ✕ - - - )) + + ); + }) )}
-
diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx index 9f320049b09..ecb8dd161f3 100644 --- a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx @@ -63,6 +63,24 @@ describe("SearchSelect", () => { expect(onValueChange).toHaveBeenLastCalledWith("team-3"); }); + it("offers the typed text as a value only when the caller accepts it", async () => { + const user = userEvent.setup(); + const onValueChange = vi.fn(); + const customValue = { hint: "Custom value", accepts: (typed: string) => typed.endsWith("*") }; + const { rerender } = render(); + const input = screen.getByRole("combobox"); + await user.click(input); + fireEvent.change(input, { target: { value: "team-*" } }); + expect(screen.queryByRole("option", { name: /team-\*/ })).not.toBeInTheDocument(); + + rerender(); + fireEvent.change(input, { target: { value: "team-4" } }); + expect(screen.queryByRole("option", { name: /team-4/ })).not.toBeInTheDocument(); + fireEvent.change(input, { target: { value: "team-*" } }); + await user.click(await screen.findByRole("option", { name: /team-\*/ })); + expect(onValueChange).toHaveBeenLastCalledWith("team-*"); + }); + it("filters the options client-side as you type", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx index 3649114826d..e139720a2ed 100644 --- a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx +++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx @@ -1,6 +1,6 @@ "use client"; -import React from "react"; +import React, { useState } from "react"; import { Combobox, ComboboxContent, @@ -19,6 +19,11 @@ export interface SearchSelectOption { icon?: React.ReactNode; } +interface CustomValue { + hint: string; + accepts: (typed: string) => boolean; +} + interface SearchSelectProps { options: SearchSelectOption[]; value?: string | null; @@ -29,6 +34,8 @@ interface SearchSelectProps { className?: string; inputId?: string; allowClear?: boolean; + customValue?: CustomValue; + reservedValues?: readonly string[]; "aria-label"?: string; id?: string; } @@ -49,13 +56,22 @@ export function SearchSelect({ className, inputId, allowClear = true, + customValue, + reservedValues = [], "aria-label": ariaLabel, id, }: SearchSelectProps) { + const [query, setQuery] = useState(""); const selected = value == null || value === "" ? null : options.find((option) => option.value === value) ?? { label: value, value }; - const items = + const known = selected !== null && !options.some((option) => option.value === selected.value) ? [selected, ...options] : options; + const typed = query.trim(); + const isNewValue = typed !== "" && !reservedValues.includes(typed) && !known.some((option) => option.value === typed); + const items = + customValue !== undefined && isNewValue && customValue.accepts(typed) + ? [...known, { label: typed, value: typed, sublabel: customValue.hint }] + : known; return ( a.value === b.value} itemToStringLabel={(item: SearchSelectOption) => item.label} filter={matchesQuery} + onInputValueChange={setQuery} disabled={disabled} > x. - * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) + * - model_max_budget: Optional[dict] - Model-specific max budget for user, keyed by model name, `*` pattern, or model access group. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) * - mcp_rpm_limit: Optional[dict] - Per-MCP-server rpm limit, keyed by MCP server name {"github": 100, "slack": 200}. Enforced for keys and teams only; values set on a user are stored but not enforced per user. @@ -18917,7 +18917,7 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) + * - model_max_budget: Optional[dict] - Model-specific max budget for user, keyed by model name, `*` pattern, or model access group. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) * - mcp_rpm_limit: Optional[dict] - Per-MCP-server rpm limit, keyed by MCP server name {"github": 100, "slack": 200}. Enforced for keys and teams only; values set on a user are stored but not enforced per user. From 41c4c3788e0e39c972f6bf4ed71451e61cbb24f8 Mon Sep 17 00:00:00 2001 From: tessellark <349031000+tessellark@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:42:51 -0700 Subject: [PATCH 12/27] test(routing): audit #46601 (#48910) * test(routing): audit #46601 Adds integration cells for the falsy budget 400, the empty and none-only declarations under drop_params, Vertex AI Claude, Databricks Claude, and the Bedrock native output retry * test(routing): expect the Anthropic default max_tokens on Bedrock Invoke chat Reads the default from litellm instead of pinning 4096, which #48502 raised to 128000 * test(routing): assert whole sent bodies in the Vertex AI and Bedrock native output cells --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../_reasoning_effort_levels_support.py | 5 + .../test_reasoning_effort_levels_wire.py | 320 +++++++++++++++++- 2 files changed, 316 insertions(+), 9 deletions(-) diff --git a/tests/integration/routing/_reasoning_effort_levels_support.py b/tests/integration/routing/_reasoning_effort_levels_support.py index 8d43695e419..d295f6e6d56 100644 --- a/tests/integration/routing/_reasoning_effort_levels_support.py +++ b/tests/integration/routing/_reasoning_effort_levels_support.py @@ -32,6 +32,7 @@ GEMINI: Final = "gemini-3-pro-preview" ZAI_GLM: Final = "glm-5" ECHOED_KEY_MODEL: Final = f"{KIMI}-echoed-key" BEDROCK_CLAUDE: Final = f"us.anthropic.{CLAUDE}" +DATABRICKS_CLAUDE: Final = f"databricks-{CLAUDE}" AZURE_API_VERSION: Final = "2025-04-01-preview" SYNTHETIC_KEY: Final = "synthetic-provider-key" _MARKER: Final = re.compile(r"marker-([0-9a-f]{32})") @@ -215,6 +216,10 @@ def backend(kind: str, wire_url: str) -> Backend: f"bedrock/invoke/{BEDROCK_CLAUDE}", MappingProxyType({"api_base": wire_url, "api_key": SYNTHETIC_KEY, "aws_region_name": "us-east-1"}), ) + case "databricks_claude": + return Backend( + f"databricks/{DATABRICKS_CLAUDE}", MappingProxyType({"api_base": wire_url, "api_key": SYNTHETIC_KEY}) + ) case _: raise AssertionError(f"unknown backend kind {kind}") diff --git a/tests/integration/routing/test_reasoning_effort_levels_wire.py b/tests/integration/routing/test_reasoning_effort_levels_wire.py index 3d88aad547c..d15867ac572 100644 --- a/tests/integration/routing/test_reasoning_effort_levels_wire.py +++ b/tests/integration/routing/test_reasoning_effort_levels_wire.py @@ -1,4 +1,5 @@ import asyncio +import json from collections.abc import Callable, Mapping from dataclasses import dataclass from types import MappingProxyType @@ -8,9 +9,11 @@ import anthropic import httpx import openai import pytest +from integration._support.anthropic_sse import message_json from integration._support.client import Gateway, eventually, list_value, object_value, string_value from integration._support.openai_wire import openai_error -from integration._support.wire import Request, wire_server +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, wire_server from integration.routing._reasoning_effort_levels_support import ( AZURE_API_VERSION, CLAUDE, @@ -34,6 +37,7 @@ from integration.routing._reasoning_effort_levels_support import ( marker_of, messages_request, new_marker, + path_of, prompt, provider_reply, responses_request, @@ -41,6 +45,8 @@ from integration.routing._reasoning_effort_levels_support import ( ) from pydantic import JsonValue +from litellm.constants import DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS + CHAT: Final = "/v1/chat/completions" MESSAGES: Final = "/v1/messages" RESPONSES: Final = "/v1/responses" @@ -49,6 +55,7 @@ _ADAPTIVE: Final[dict[str, JsonValue]] = {"type": "adaptive"} _STRING_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": "2048"} _XHIGH_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 9000} _BOOL_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": True} +_FALSE_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": False} _NULL_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": None} _LIST_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": [2048]} _EMPTY_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": ""} @@ -67,6 +74,8 @@ _KIMI_INFO: Final[Mapping[str, JsonValue]] = MappingProxyType({"reasoning_effort _CLAUDE_INFO: Final[Mapping[str, JsonValue]] = MappingProxyType({"reasoning_effort_levels": list(CLAUDE_LEVELS)}) _GPT_INFO: Final[Mapping[str, JsonValue]] = MappingProxyType({"reasoning_effort_levels": list(GPT_LEVELS)}) _UNDECLARED: Final[Mapping[str, JsonValue]] = MappingProxyType({}) +_EMPTY_DECLARATION: Final[Mapping[str, JsonValue]] = MappingProxyType({"reasoning_effort_levels": []}) +_NONE_ONLY_DECLARATION: Final[Mapping[str, JsonValue]] = MappingProxyType({"reasoning_effort_levels": ["none"]}) _ECHOED_MAP_KEY: Final[Mapping[str, JsonValue]] = MappingProxyType( {"key": "azure/chat-latest", "litellm_provider": "azure"} ) @@ -134,7 +143,7 @@ def _invoke_chat(effort: str, thinking: Mapping[str, JsonValue], betas: tuple[st "messages": [{"role": "user", "content": [{"type": "text", "text": prompt(marker)}]}], "thinking": dict(thinking), "output_config": {"effort": effort}, - "max_tokens": 4096, + "max_tokens": DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS, "anthropic_version": "bedrock-2023-05-31", "anthropic_beta": list(betas), } @@ -144,6 +153,14 @@ def _with_output_config(expected: Expected, output_config: JsonValue) -> Expecte return lambda marker: {**expected(marker), "output_config": output_config} +def _with_fields(expected: Expected, fields: Mapping[str, JsonValue]) -> Expected: + return lambda marker: {**expected(marker), **fields} + + +def _without(expected: Expected, field: str) -> Expected: + return lambda marker: {key: value for key, value in expected(marker).items() if key != field} + + def _converse_chat(effort: str, thinking: Mapping[str, JsonValue]) -> Expected: return lambda marker: { "messages": [{"role": "user", "content": [{"text": prompt(marker)}]}], @@ -320,6 +337,7 @@ _OPENAI_RESPONSES: Final = "/v1/responses" _AZURE_RESPONSES: Final = f"/openai/responses?api-version={AZURE_API_VERSION}" _UNSUPPORTED: Final = "does not support parameters" _GEMINI: Final = f"/models/{GEMINI}:generateContent" +_DATABRICKS: Final = "/chat/completions" CELLS: Final[Mapping[str, Cell]] = MappingProxyType( { @@ -1750,6 +1768,139 @@ CELLS: Final[Mapping[str, Cell]] = MappingProxyType( _anthropic_chat_without_effort(_ADAPTIVE), _ANTHROPIC, ), + "chat-anthropic-false-budget-400": Cell( + CHAT, "anthropic", _CLAUDE_INFO, {"thinking": _FALSE_BUDGET}, None, _ANTHROPIC, 400, _NON_NUMERIC_BUDGET + ), + "chat-anthropic-undeclared-null-budget-400": Cell( + CHAT, "anthropic", _UNDECLARED, {"thinking": _NULL_BUDGET}, None, _ANTHROPIC, 400, _NON_NUMERIC_BUDGET + ), + "chat-anthropic-undeclared-false-budget-drop-params-sends-adaptive-without-effort": Cell( + CHAT, + "anthropic", + _UNDECLARED, + {"thinking": _FALSE_BUDGET, "drop_params": True}, + _anthropic_chat_without_effort(_ADAPTIVE), + _ANTHROPIC, + ), + "chat-anthropic-undeclared-missing-budget-sends-low": Cell( + CHAT, + "anthropic", + _UNDECLARED, + {"thinking": _MISSING_BUDGET}, + _anthropic_chat("low", _ADAPTIVE), + _ANTHROPIC, + ), + "messages-anthropic-undeclared-false-budget-400": Cell( + MESSAGES, "anthropic", _UNDECLARED, {"thinking": _FALSE_BUDGET}, None, _ANTHROPIC, 400, _NON_NUMERIC_BUDGET + ), + "messages-anthropic-empty-string-budget-400": Cell( + MESSAGES, "anthropic", _CLAUDE_INFO, {"thinking": _EMPTY_BUDGET}, None, _ANTHROPIC, 400, _NON_NUMERIC_BUDGET + ), + "messages-anthropic-undeclared-null-budget-drop-params-sends-adaptive-without-effort": Cell( + MESSAGES, + "anthropic", + _UNDECLARED, + {"thinking": _NULL_BUDGET, "drop_params": True}, + _native_messages(None, _ADAPTIVE), + _ANTHROPIC, + ), + "responses-anthropic-false-budget-400": Cell( + RESPONSES, + "anthropic", + _CLAUDE_INFO, + {"thinking": _FALSE_BUDGET}, + None, + _ANTHROPIC, + 400, + _NON_NUMERIC_BUDGET, + ), + "chat-converse-undeclared-false-budget-400": Cell( + CHAT, + "bedrock_converse", + _UNDECLARED, + {"thinking": _FALSE_BUDGET}, + None, + _CONVERSE, + 400, + _NON_NUMERIC_BUDGET, + ), + "messages-invoke-undeclared-null-budget-400": Cell( + MESSAGES, "bedrock_invoke", _UNDECLARED, {"thinking": _NULL_BUDGET}, None, _INVOKE, 400, _NON_NUMERIC_BUDGET + ), + "chat-databricks-claude-undeclared-false-budget-400": Cell( + CHAT, + "databricks_claude", + _UNDECLARED, + {"thinking": _FALSE_BUDGET}, + None, + _DATABRICKS, + 400, + _NON_NUMERIC_BUDGET, + ), + "chat-anthropic-empty-declaration-budget-drop-params-sends-adaptive-without-effort": Cell( + CHAT, + "anthropic", + _EMPTY_DECLARATION, + {"thinking": _BUDGET, "drop_params": True}, + _anthropic_chat_without_effort(_ADAPTIVE), + _ANTHROPIC, + ), + "chat-anthropic-none-only-declaration-budget-drop-params-sends-adaptive-without-effort": Cell( + CHAT, + "anthropic", + _NONE_ONLY_DECLARATION, + {"thinking": _BUDGET, "drop_params": True}, + _anthropic_chat_without_effort(_ADAPTIVE), + _ANTHROPIC, + ), + "chat-anthropic-empty-declaration-budget-keeps-its-effort": Cell( + CHAT, + "anthropic", + _EMPTY_DECLARATION, + {"thinking": _BUDGET}, + _anthropic_chat("medium", _ADAPTIVE), + _ANTHROPIC, + ), + "chat-converse-empty-declaration-budget-drop-params-sends-adaptive-without-effort": Cell( + CHAT, + "bedrock_converse", + _EMPTY_DECLARATION, + {"thinking": _BUDGET, "drop_params": True}, + _converse_chat_without_effort(_ADAPTIVE), + _CONVERSE, + ), + "messages-anthropic-empty-declaration-budget-drop-params-sends-adaptive-without-effort": Cell( + MESSAGES, + "anthropic", + _EMPTY_DECLARATION, + {"thinking": _BUDGET, "drop_params": True}, + _native_messages(None, _ADAPTIVE), + _ANTHROPIC, + ), + "messages-anthropic-none-only-declaration-budget-drop-params-sends-adaptive-without-effort": Cell( + MESSAGES, + "anthropic", + _NONE_ONLY_DECLARATION, + {"thinking": _BUDGET, "drop_params": True}, + _native_messages(None, _ADAPTIVE), + _ANTHROPIC, + ), + "messages-anthropic-empty-declaration-budget-keeps-its-effort": Cell( + MESSAGES, + "anthropic", + _EMPTY_DECLARATION, + {"thinking": _BUDGET}, + _native_messages({"effort": "medium"}, _ADAPTIVE), + _ANTHROPIC, + ), + "messages-converse-empty-declaration-budget-drop-params-sends-adaptive-without-effort": Cell( + MESSAGES, + "bedrock_converse", + _EMPTY_DECLARATION, + {"thinking": _BUDGET, "drop_params": True}, + _converse_messages(None), + _CONVERSE, + ), } ) @@ -1815,20 +1966,65 @@ def _bedrock_native_schema(body: Mapping[str, JsonValue], *, invoke: bool) -> Js return JSON_OBJECT.validate_json(string_value(json_schema["schema"])) +_CONVERSE_NATIVE_OUTPUT: Final[dict[str, JsonValue]] = { + "textFormat": { + "type": "json_schema", + "structure": {"jsonSchema": {"name": "answer", "schema": json.dumps(_NATIVE_OUTPUT_SCHEMA)}}, + } +} +_INVOKE_NATIVE_OUTPUT: Final[dict[str, JsonValue]] = { + "effort": "high", + "format": {"type": "json_schema", "schema": _NATIVE_OUTPUT_SCHEMA}, +} +_JSON_TOOL: Final = "json_tool_call" +_CONVERSE_TOOL_SCHEMA: Final[dict[str, JsonValue]] = { + key: value for key, value in _NATIVE_OUTPUT_SCHEMA.items() if key != "additionalProperties" +} +_CONVERSE_WITH_NATIVE_OUTPUT: Final = _with_fields( + _converse_chat("high", _ADAPTIVE), {"outputConfig": _CONVERSE_NATIVE_OUTPUT} +) +_INVOKE_WITH_NATIVE_OUTPUT: Final = _with_output_config( + _invoke_chat("high", _ADAPTIVE, ("structured-outputs-2025-11-13",)), _INVOKE_NATIVE_OUTPUT +) +_CONVERSE_WITH_JSON_TOOL: Final = _with_fields( + _converse_chat("high", _ADAPTIVE), + { + "toolConfig": { + "tools": [ + { + "toolSpec": { + "name": _JSON_TOOL, + "description": _JSON_TOOL, + "inputSchema": {"json": _CONVERSE_TOOL_SCHEMA}, + } + } + ] + } + }, +) +_INVOKE_WITH_JSON_TOOL: Final = _with_fields( + _without(_invoke_chat("high", _ADAPTIVE, ()), "anthropic_beta"), + {"tools": [{"name": _JSON_TOOL, "input_schema": _NATIVE_OUTPUT_SCHEMA}]}, +) + + @pytest.mark.parametrize( - ("kind", "target", "fields"), + ("kind", "target", "fields", "expected"), [ - pytest.param("bedrock_converse", _CONVERSE, {"thinking": _BUDGET}, id="converse-budget"), + pytest.param( + "bedrock_converse", _CONVERSE, {"thinking": _BUDGET}, _CONVERSE_WITH_NATIVE_OUTPUT, id="converse-budget" + ), pytest.param( "bedrock_invoke", _INVOKE, {"thinking": _BUDGET, "output_config": {"effort": "medium"}}, + _INVOKE_WITH_NATIVE_OUTPUT, id="invoke-output-config-beside-budget", ), ], ) def test_a_deployment_opted_into_bedrock_native_output_keeps_it_beside_its_declared_level( - gateway: Gateway, kind: str, target: str, fields: Mapping[str, JsonValue] + gateway: Gateway, kind: str, target: str, fields: Mapping[str, JsonValue], expected: Expected ) -> None: marker: Final = new_marker() response_format: Final[dict[str, JsonValue]] = { @@ -1844,10 +2040,116 @@ def test_a_deployment_opted_into_bedrock_native_output_keeps_it_beside_its_decla assert response.status_code == 200, response.text[:2000] assert answer(marker) in response.text, response.text[:2000] assert [(request.method, request.target) for request in received] == [("POST", target)] - body: Final = body_of(received[0]) - assert _bedrock_native_schema(body, invoke=target == _INVOKE) == _NATIVE_OUTPUT_SCHEMA, body - assert {"toolConfig", "tools"}.isdisjoint(body), body - assert sent_effort(received[0]) == "high", body + assert body_of(received[0]) == expected(marker) + + +_NATIVE_OUTPUT_REJECTION: Final = ( + "The model returned the following errors: output_config.format: Extra inputs are not permitted" +) + + +def _rejecting_native_output(request: Request) -> Reply: + native: Final = _bedrock_native_schema(body_of(request), invoke=path_of(request).endswith("/invoke")) + if native is None: + return provider_reply(request) + return Reply(status=400, body=json.dumps({"message": _NATIVE_OUTPUT_REJECTION}).encode()) + + +@pytest.mark.parametrize( + ("kind", "target", "rejected", "retried"), + [ + pytest.param( + "bedrock_converse", _CONVERSE, _CONVERSE_WITH_NATIVE_OUTPUT, _CONVERSE_WITH_JSON_TOOL, id="converse" + ), + pytest.param("bedrock_invoke", _INVOKE, _INVOKE_WITH_NATIVE_OUTPUT, _INVOKE_WITH_JSON_TOOL, id="invoke"), + ], +) +def test_the_retry_after_bedrock_rejects_native_output_keeps_the_declared_level( + gateway: Gateway, kind: str, target: str, rejected: Expected, retried: Expected +) -> None: + marker: Final = new_marker() + response_format: Final[dict[str, JsonValue]] = { + "type": "json_schema", + "json_schema": {"name": "answer", "schema": _NATIVE_OUTPUT_SCHEMA}, + } + with wire_server(_rejecting_native_output) as wire, gateway.scenario() as scenario: + model: Final = deployment(scenario, kind, wire, _CLAUDE_INFO_WITH_NATIVE_OUTPUT) + response: Final = gateway.request( + "POST", CHAT, chat_request(model, marker, response_format=response_format, thinking=_BUDGET) + ) + received: Final = wire.drain() + assert response.status_code == 200, response.text[:2000] + assert [(request.method, request.target) for request in received] == [("POST", target), ("POST", target)] + assert [body_of(request) for request in received] == [rejected(marker), retried(marker)] + + +_VERTEX_PROJECT: Final = "scripted-project" +_VERTEX_LOCATION: Final = "us-east5" +_VERTEX_TOKEN: Final = "/_oauth/token" +_VERTEX_RAW_PREDICT: Final = ( + f"/v1/projects/{_VERTEX_PROJECT}/locations/{_VERTEX_LOCATION}/publishers/anthropic/models/{CLAUDE}:rawPredict" +) +_VERTEX_ACCESS_TOKEN: Final = json.dumps({"access_token": "scripted", "token_type": "Bearer", "expires_in": 3600}) + + +def _vertex(expected: Expected) -> Expected: + return _with_fields(_without(expected, "model"), {"anthropic_version": "vertex-2023-10-16"}) + + +def _vertex_reply(request: Request) -> Reply: + if request.target == _VERTEX_TOKEN: + return Reply(body=_VERTEX_ACCESS_TOKEN.encode()) + marker: Final = marker_of(request) + return Reply(body=message_json(f"msg_{marker}", CLAUDE, answer(marker))) + + +@pytest.mark.parametrize( + ("route", "fields", "expected", "status"), + [ + pytest.param( + CHAT, + {"reasoning_effort": "medium"}, + _vertex(_anthropic_chat("high", {"type": "adaptive", "display": "summarized"})), + 200, + id="chat-medium-to-high", + ), + pytest.param( + MESSAGES, + {"thinking": _BUDGET}, + _vertex(_native_messages({"effort": "high"}, _ADAPTIVE)), + 200, + id="messages-budget-to-high", + ), + pytest.param(CHAT, {"thinking": _FALSE_BUDGET}, None, 400, id="chat-false-budget-400"), + ], +) +def test_a_vertex_ai_claude_deployment_gets_one_of_its_declared_levels( + gateway: Gateway, route: str, fields: dict[str, JsonValue], expected: Expected | None, status: int +) -> None: + marker: Final = new_marker() + with wire_server(_vertex_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"vertex_ai/{CLAUDE}", + api_base=wire.url, + api_key=None, + vertex_project=_VERTEX_PROJECT, + vertex_location=_VERTEX_LOCATION, + vertex_credentials=service_account_json(_VERTEX_PROJECT, wire.url), + model_info=dict(_CLAUDE_INFO), + ) + body: Final = ( + chat_request(model, marker, **fields) if route == CHAT else messages_request(model, marker, **fields) + ) + response: Final = gateway.request("POST", route, body) + received: Final = tuple(request for request in wire.drain() if request.target != _VERTEX_TOKEN) + assert response.status_code == status, response.text[:2000] + if expected is None: + assert _NON_NUMERIC_BUDGET in response.text, response.text[:2000] + assert received == (), [request.body[:300] for request in received] + return + assert answer(marker) in response.text, response.text[:2000] + assert [(request.method, request.target) for request in received] == [("POST", _VERTEX_RAW_PREDICT)] + assert body_of(received[0]) == expected(marker) def _single(received: tuple[Request, ...], marker: str) -> Request: From ddb8a14ee97e3abb6c251edfd3a4ba53467688db Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:44:43 -0700 Subject: [PATCH 13/27] feat(skills): add a litellm-setup agent skill (#47731) * feat(skills): add a litellm-setup agent skill * fix(skills): run the quickstart for Docker with Postgres in litellm-setup Use scripts/quickstart.sh for the Docker plus Postgres path instead of a hand-rolled Postgres container, keep uv with the user's own Postgres, load .env as data instead of sourcing it, pin the uv Python range, chmod an existing .env, list bedrock_converse and vertex_ai rows in the cost map lookup, ask before editing the shell profile, and stop the health wait when the proxy exits * fix(skills): guide empty Postgres, docker sudo, and key fixes in litellm-setup Ask for a new empty database on the uv path and DIRECT_URL behind a pooler, add the baseline error to troubleshooting, and drop inherited database settings in the loader line. Check python3 and lsof, explain Docker's permission denied, give quickstart commands by the folder's path, and fix provider keys by recreating the container or deleting the quickstart model. Drop the structure-only test. * fix(skills): keep shell profiles untouched when litellm-setup installs uv The uv installer edits shell profiles unless UV_NO_MODIFY_PATH is set, so the skill now sets it and puts ~/.local/bin on PATH for later commands. It also says litellm --setup runs in a separate terminal and how to retire the shared master-key UI login * fix(skills): offer litellm --setup only on the uv path The Docker paths install no host litellm CLI, and the old wording also passed the skill's own step number on to the user * fix(skills): ask before using an env provider key, print model names config.yaml accepts, probe the port with curl * fix(skills): state the env key default in the question, chmod the wizard config, keep uv's bin dir on PATH * fix(skills): keep litellm-setup on its own .env and install when the shell has others * fix(skills): read the setup wizard's config without printing keys and hand off the exact restart command * fix(skills): hide the database-access dev skill from end-user skill installs * fix(skills): give the litellm --setup path a restart command that keeps its config * docs(skills): point litellm --setup users to the master key in litellm_config.yaml * fix(skills): hide every non-end-user skill, use Azure deployment names, keep the restart block out of the user's shell * test(skills): find non-end-user skills in-process instead of through git * docs(skills): price Azure deployments with base_model and put uv on PATH in the start block --- .agents/skills/database-access/SKILL.md | 2 + .../.agents/skills/rust-error/SKILL.md | 2 + .../.agents/skills/rust-gateway/SKILL.md | 2 + .../.agents/skills/rust-string-enums/SKILL.md | 2 + .../.agents/skills/rust-tracing/SKILL.md | 2 + skills/litellm-setup/SKILL.md | 205 ++++++++++++++++++ .../slack-gif-creator/SKILL.md | 2 + .../test-delete-skill/SKILL.md | 2 + .../test-skill-litellm/SKILL.md | 2 + tests/unit/test_repo_agent_skills.py | 68 ++++++ .../.agents/skills/url-state/SKILL.md | 2 + 11 files changed, 291 insertions(+) create mode 100644 skills/litellm-setup/SKILL.md create mode 100644 tests/unit/test_repo_agent_skills.py diff --git a/.agents/skills/database-access/SKILL.md b/.agents/skills/database-access/SKILL.md index 796dfafe4f6..13fb7b7e286 100644 --- a/.agents/skills/database-access/SKILL.md +++ b/.agents/skills/database-access/SKILL.md @@ -1,6 +1,8 @@ --- name: database-access description: Add database queries, extract existing Prisma calls into repositories, or explicitly port queries to Rust and SQLx without expanding migration scope +metadata: + internal: true --- # Database access diff --git a/litellm-rust/.agents/skills/rust-error/SKILL.md b/litellm-rust/.agents/skills/rust-error/SKILL.md index fd5d6222287..74da1ecbcf4 100644 --- a/litellm-rust/.agents/skills/rust-error/SKILL.md +++ b/litellm-rust/.agents/skills/rust-error/SKILL.md @@ -1,6 +1,8 @@ --- name: rust-error description: Define, wrap, convert, classify, report or test Rust error types in litellm-rust with thiserror. Use when adding an error type or variant, mapping one crate's error into another, turning an error into a gateway response or Python exception, deciding retry or fault from an error, or replacing call sites that build errors from strings +metadata: + internal: true --- # Rust errors diff --git a/litellm-rust/.agents/skills/rust-gateway/SKILL.md b/litellm-rust/.agents/skills/rust-gateway/SKILL.md index 66b93ee128d..4fee0b54a68 100644 --- a/litellm-rust/.agents/skills/rust-gateway/SKILL.md +++ b/litellm-rust/.agents/skills/rust-gateway/SKILL.md @@ -1,6 +1,8 @@ --- name: rust-gateway description: Add or change an HTTP route, extractor, response envelope, stream or router mount in the litellm-rust gateway crates (crates/gateway and crates/gateway-*). Use when adding an endpoint, parsing a request body, choosing an error envelope for an API surface, relaying SSE, or writing a router-level test +metadata: + internal: true --- # Rust gateway diff --git a/litellm-rust/.agents/skills/rust-string-enums/SKILL.md b/litellm-rust/.agents/skills/rust-string-enums/SKILL.md index 5fc067790a0..352f6427dd9 100644 --- a/litellm-rust/.agents/skills/rust-string-enums/SKILL.md +++ b/litellm-rust/.agents/skills/rust-string-enums/SKILL.md @@ -1,6 +1,8 @@ --- name: rust-string-enums description: Define or refactor Rust string-valued enums and their Serde adapters in litellm-rust, using Strum and serde_with while preserving parsing, wire values, and schemas +metadata: + internal: true --- # Rust string enums diff --git a/litellm-rust/.agents/skills/rust-tracing/SKILL.md b/litellm-rust/.agents/skills/rust-tracing/SKILL.md index c1c7f43c456..a8da3f3bb14 100644 --- a/litellm-rust/.agents/skills/rust-tracing/SKILL.md +++ b/litellm-rust/.agents/skills/rust-tracing/SKILL.md @@ -1,6 +1,8 @@ --- name: rust-tracing description: Add or change Rust diagnostic tracing in litellm-rust, including route spans, subscriber layers, and Python logger delivery +metadata: + internal: true --- # Rust tracing diff --git a/skills/litellm-setup/SKILL.md b/skills/litellm-setup/SKILL.md new file mode 100644 index 00000000000..7c824751d6f --- /dev/null +++ b/skills/litellm-setup/SKILL.md @@ -0,0 +1,205 @@ +--- +name: litellm-setup +description: Set up a self-hosted LiteLLM proxy, the OpenAI-compatible LLM gateway, on this machine, from install through provider keys, config.yaml, a master key, optional Postgres and Admin UI, and a verified chat completion. Use when the user asks to set up LiteLLM, install the LiteLLM proxy, self-host an LLM gateway, or put one endpoint in front of OpenAI, Anthropic, Gemini, Azure, Bedrock, Ollama, or other providers. Not for using the LiteLLM Python SDK inside an app +--- + +# Set up a LiteLLM proxy + +Take the user from nothing to a running LiteLLM proxy that has answered one chat completion. Run every command yourself unless a step says the user does it + +Never print a secret. Provider keys, the master key, the salt key, the database URL, and virtual keys live only in `.env`, and `config.yaml` reads them through `os.environ/NAME`. Check that a value is there with `grep -c '^NAME=.' .env`, never with `cat .env` or `echo "$NAME"`. Load `.env` into a shell with this line, which exports each line as it is, so a `&` or `$` in a value stays literal, and first drops any database settings the shell already had and sets `LITELLM_MODE=PRODUCTION`, which stops `litellm` from also loading a `.env` it finds above its install folder, such as `~/.env`, so only this `.env` picks the database. The steps that need it start with it: + +```bash +unset DATABASE_URL DIRECT_URL DATABASE_HOST; export LITELLM_MODE=PRODUCTION; while IFS= read -r line || [ -n "$line" ]; do case "$line" in [A-Za-z_]*=*) export "$line" ;; esac; done < .env +``` + +## 1. Ask once + +Ask everything in one message, with defaults the user can accept by saying "go": + +- Providers and models. Default is OpenAI with its newest flagship model, using the `OPENAI_API_KEY` in your environment when `[ -n "$OPENAI_API_KEY" ] && echo set` says it is there. Run that check before asking, and say in the question that the default uses that variable, naming it but never its value, since a key in your environment can belong to another account +- Whether they want virtual keys, teams, spend tracking, or the Admin UI. Those need Postgres, plain routing does not. Default is no +- uv (default) or Docker. With Postgres, Docker runs the repo's quickstart, which brings its own database, and uv needs a Postgres server they already have, with a new, empty database for LiteLLM, so someone with their own Postgres server picks uv +- Folder and port. Defaults are `./litellm-proxy` (`./litellm-gateway` for the quickstart) and 4000 + +Then follow the steps for their answers. Plain routing skips steps 3 and 6. The quickstart is steps 2, 3, 8, and 9. uv with their own Postgres skips step 3 + +In the same message, say that someone who picks uv and would rather answer prompts can run `litellm --setup` in a separate terminal window once LiteLLM is installed, not through Claude Code. Never run it yourself, since it needs a real terminal and without one prints "Setup cancelled." and writes nothing. Give them the command as `LITELLM_MODE=PRODUCTION /litellm --setup`, with `` replaced by the folder `uv tool dir --bin` prints, to run from the folder they want the config in. It writes `litellm_config.yaml` there with the provider keys and master key in plain text and offers to start the proxy, which then listens on every network interface and stops with Ctrl+C in that window. When the user is done, run this in that folder. It makes the file readable only by them, keeps it out of git (the wizard does neither), and prints only the model names. Never print that file or run `grep`, `sed`, or `cat` on it: + +```bash +chmod 600 litellm_config.yaml; grep -qx 'litellm_config.yaml' .gitignore 2>/dev/null || echo 'litellm_config.yaml' >> .gitignore +"$(uv tool dir)/litellm/bin/python" -I -c 'import yaml; print(*(m["model_name"] for m in yaml.safe_load(open("litellm_config.yaml"))["model_list"]), sep="\n")' +``` + +Then skip to step 8 and run it against their proxy, starting that shell command with this line instead of the loader line, which reads the master key without printing it: + +```bash +export LITELLM_MASTER_KEY="$("$(uv tool dir)/litellm/bin/python" -I -c 'import yaml; print(yaml.safe_load(open("litellm_config.yaml"))["general_settings"]["master_key"])')" +``` + +## 2. Check prerequisites + +```bash +uv --version; docker info --format '{{.ServerVersion}}'; docker compose version; curl --version | head -1; openssl version; command -v python3 +``` + +On native Windows, ask the user to run Claude Code inside WSL, since every command here is POSIX shell. Every path needs curl, openssl, and python3, so ask the user to install any that is missing. The uv path needs uv, which downloads a supported Python (LiteLLM supports 3.10 to 3.14) when the `--python` range in step 4 rules out the one on the machine. If uv is missing, offer to install it, and only after the user says yes run `curl -LsSf https://astral.sh/uv/install.sh | env UV_NO_MODIFY_PATH=1 sh`, which leaves their shell profile alone (https://docs.astral.sh/uv/getting-started/installation/). It puts uv in `~/.local/bin`, so start every later command that runs `uv` or `litellm` with `export PATH="$HOME/.local/bin:$PATH";`. The Docker path needs a running Docker daemon, and the quickstart also needs Docker Compose v2. If `docker info` says `permission denied`, the user's account cannot use Docker without sudo. Offer the uv path, or ask them to add themselves to the `docker` group the way https://docs.docker.com/engine/install/linux-postinstall/ shows and start a new login session before you check again + +Check the port is free with `curl -s -o /dev/null --max-time 2 http://127.0.0.1:4000; echo $?`, which prints `7` (could not connect) only when nothing listens there. Don't use `lsof` for this, since it misses ports that root or Docker hold. If it is taken, pick another port and use it everywhere below, because a proxy asked for a busy 4000 silently moves to a random port + +## 3. Postgres with Docker: run the quickstart + +The repo's quickstart starts the gateway, Postgres, and the Admin UI in Docker, listening on this machine only, and puts a `litellm-gateway` command in that folder that stops, starts, updates, or removes them. It asks nothing when it has no terminal. It writes the master key, salt key, and database password to its own `.env` and never prints them. Download it, then run it with the folder and port from step 1. It returns once the gateway is ready: + +```bash +curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/quickstart.sh -o quickstart.sh +LITELLM_DIR="$PWD/litellm-gateway" LITELLM_PORT=4000 sh quickstart.sh --yes +cd litellm-gateway +``` + +Models live in the database here, so add each one through the API instead of a `config.yaml`, naming it the way step 5 says. Pipe the request body in, so the provider key never shows up in output or on a command line. Unless the user agreed to the key in your environment, as step 5 says, ask them to add the line `OPENAI_API_KEY=` to this `.env` themselves. Run the loader line first: + +```bash +python3 -c 'import json, os, sys; json.dump({"model_name": "gpt-6.1-sol", "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": os.environ["OPENAI_API_KEY"]}}, sys.stdout)' \ + | curl -sS -o /dev/null -w '%{http_code}\n' http://127.0.0.1:4000/model/new \ + -H "Authorization: Bearer $LITELLM_MASTER_KEY" -H "Content-Type: application/json" -d @- +``` + +Run that in one shell command that starts with the loader line, since the master key is in `.env`. `200` means the model was added. Then go to step 8 + +## 4. Install + +uv: + +```bash +uv tool install --python '>=3.10,<3.15' 'litellm[proxy]' +export PATH="$(uv tool dir --bin):$PATH"; litellm --version +``` + +With their own Postgres, install `'litellm[proxy,extra_proxy]'` instead, with the same `--python` range, which adds the Prisma client the proxy uses for the database. Running `uv tool install` again with the new extras replaces the old install. Start every later command that runs `litellm` with `export PATH="$(uv tool dir --bin):$PATH";`, so this install runs even when the shell has no `litellm` or an older one from a venv or `pip install litellm`. `uv tool update-shell` puts it on the PATH of new shells by editing the user's shell profile, so run it only after the user says yes + +Docker: + +```bash +docker pull docker.litellm.ai/berriai/litellm:main-stable +``` + +## 5. Write `.env` and `config.yaml` + +Run every command from here on in the folder from step 1. Create `.env` readable only by the user, keep it out of git, and add a master key unless it already has one. The proxy refuses to start without one, and `sk-` is the format of every key it issues: + +```bash +mkdir -p litellm-proxy && cd litellm-proxy +touch .env && chmod 600 .env; grep -qx '.env' .gitignore 2>/dev/null || echo '.env' >> .gitignore +grep -q '^LITELLM_MASTER_KEY=.' .env || printf 'LITELLM_MASTER_KEY=sk-%s\n' "$(openssl rand -hex 32)" >> .env +``` + +Copy a provider key that is already in your environment only after the user agreed to that variable, in their step 1 answer or when you ask now, and copy it without printing it, as in `printf 'OPENAI_API_KEY=%s\n' "$OPENAI_API_KEY" >> .env`. Otherwise ask the user to add the line `OPENAI_API_KEY=` to `.env` themselves and tell you when it is there. If they paste a key into the chat anyway, write it to `.env` and never repeat it. Keep one `NAME=value` per line with no quotes, so `docker --env-file` and the loader line read the same value + +Confirm each model's current LiteLLM name in the cost map instead of trusting memory, since model names go stale fast. This prints the chat models of the provider named last (`openai`, `anthropic`, `gemini`, `azure`, `bedrock`, `vertex_ai`, and so on) the way `config.yaml` takes them, with the provider in front. It matches by prefix, so `bedrock` also lists the newer Bedrock models and `vertex_ai` lists every Vertex family. For Ollama, use a model from `ollama list` instead: + +```bash +curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json | python3 -c ' +import json, sys +for name, row in json.load(sys.stdin).items(): + provider = str(row.get("litellm_provider", "")) if isinstance(row, dict) else "" + if provider.startswith(sys.argv[1]) and row.get("mode") == "chat": + prefix = "bedrock" if provider == "bedrock_converse" else "vertex_ai" if provider.startswith("vertex_ai") else provider + print(name if name.startswith(prefix + "/") else prefix + "/" + name) +' openai +``` + +In `config.yaml`, `model` is a name from that list, as in `openai/gpt-6.1-sol` or `bedrock/us.anthropic.`, and `model_name` is what clients will send. If the folder already has a `config.yaml`, show the user its `model_name` lines and ask before replacing it. This example uses the newest GPT model in the cost map when this skill was written, so replace it with what the user picked: + +```yaml +model_list: + - model_name: gpt-6.1-sol + litellm_params: + model: openai/gpt-6.1-sol + api_key: os.environ/OPENAI_API_KEY + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY +``` + +Add one entry per model. On Azure, ask the user for each deployment's name and the model it runs. Use `azure/` as `model`, since Azure routes by deployment and a name from the lookup works only when the deployment has that same name, and add `model_info:` with `base_model: azure/` to the entry, since LiteLLM prices each call by that name. Providers that need more than a key, like Azure or Bedrock, list their `litellm_params` on their page under https://docs.litellm.ai/docs/providers. Point every secret at `.env` through `os.environ/NAME` the same way + +## 6. Their own Postgres, on the uv path + +Ask the user to create a new, empty database for LiteLLM on their server (`CREATE DATABASE litellm;`) and add `DATABASE_URL=postgresql://...` for it to `.env` themselves. The proxy creates its tables there on first start and stops on a database that already holds other tables. If that URL goes through a transaction pooler such as PgBouncer, they also add `DIRECT_URL` for the same database without the pooler, which the migrations use. Then add a salt key. It encrypts the provider credentials the proxy stores in the database and falls back to the master key when unset, so set it once and never change it: + +```bash +grep -q '^LITELLM_SALT_KEY=.' .env || printf 'LITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" >> .env +``` + +Generate the Prisma client once inside the tool's environment. A pip or uv install doesn't ship it generated (Docker images do): + +```bash +tool="$(uv tool dir)/litellm" +schema=$("$tool/bin/python" -I -c 'import importlib.util, os; print(os.path.join(importlib.util.find_spec("litellm").submodule_search_locations[0], "proxy", "schema.prisma"))') +PATH="$tool/bin:$PATH" prisma generate --schema "$schema" +``` + +## 7. Start the proxy + +Bind to `127.0.0.1` unless the user wants other machines to reach it + +uv, in one shell command that starts with the loader line, since the installed `litellm` does not read `.env` by itself: + +```bash +export PATH="$HOME/.local/bin:$PATH"; export PATH="$(uv tool dir --bin):$PATH" +nohup litellm --config config.yaml --host 127.0.0.1 --port 4000 > litellm.log 2>&1 & +echo $! > litellm.pid +``` + +Docker: + +```bash +docker run -d --name litellm --env-file .env -v "$PWD/config.yaml:/app/config.yaml:ro" -p 127.0.0.1:4000:4000 \ + docker.litellm.ai/berriai/litellm:main-stable --config /app/config.yaml --port 4000 +``` + +Wait on the health route, stopping early if the proxy exits. The first start with Postgres applies the database migrations and takes longer. Define `alive` for the path you took, `alive() { kill -0 "$(cat litellm.pid)" 2>/dev/null; }` for uv or `alive() { [ "$(docker inspect -f '{{.State.Running}}' litellm 2>/dev/null)" = true ]; }` for Docker, in the same shell command as the loop: + +```bash +for i in $(seq 1 180); do curl -fsS http://127.0.0.1:4000/health/liveliness >/dev/null 2>&1 && break; alive || break; sleep 1; done +curl -sS http://127.0.0.1:4000/health/readiness +``` + +Readiness reports `"db":"connected"` with Postgres and `"db":"Not connected"` without it. If the proxy never comes up, read the end of `litellm.log` (or `docker logs litellm`) and check the troubleshooting list + +## 8. Verify with a chat completion + +Use a `model_name` from `config.yaml` (or step 3), in one shell command that starts with the loader line: + +```bash +curl -sS http://127.0.0.1:4000/v1/chat/completions \ + -H "Authorization: Bearer $LITELLM_MASTER_KEY" -H "Content-Type: application/json" \ + -d '{"model": "gpt-6.1-sol", "messages": [{"role": "user", "content": "Say hello in five words."}]}' +``` + +Show the user the reply in `choices[0].message.content`. An error here means the setup is not done, so fix it before handing off + +## 9. Hand off + +Tell the user, without printing any secret: + +- The base URL is `http://127.0.0.1:4000`. Any OpenAI-compatible client works with it as the base URL and the master key or a virtual key as the API key +- The master key is `LITELLM_MASTER_KEY` in the folder's `.env`, or `general_settings.master_key` in `litellm_config.yaml` when they used `litellm --setup` +- To stop it, `kill "$(cat litellm.pid)"` (uv), `docker stop litellm` (Docker), `/litellm-gateway stop` (quickstart), or Ctrl+C in its window (`litellm --setup`). To start it again, `docker start litellm`, `/litellm-gateway start`, on uv the whole step 7 command as you ran it, loader line and PATH export included, wrapped in `( ... )` so their shell keeps its own settings, shown as a code block and never paraphrased, since a new shell has neither, or for `litellm --setup` `LITELLM_MODE=PRODUCTION /litellm --config litellm_config.yaml --host 127.0.0.1 --port ` in a terminal window in that folder, with `` and `` filled in, since running the wizard again writes a new config with a new master key. `update` and `uninstall` update or remove a quickstart install. Give the quickstart commands with the folder's full path, since a bare `litellm-gateway` is on PATH only when `~/.local/bin` is, and can belong to an earlier install +- With Postgres, the Admin UI is at `http://127.0.0.1:4000/ui` (user `admin`, password the master key), and `POST /key/generate` with the master key creates virtual keys with their own budgets and model lists. Without Postgres, both need step 3 or 6 first +- The Admin UI warns that the master-key login is a shared secret, which is fine while only this machine reaches the proxy. Before anyone else can reach it, invite an admin with their own password on the Internal Users page, then set `general_settings.disable_env_credential_login: true` in the proxy config +- To manage the running proxy from a coding agent (keys, teams, users, models, MCP servers), install `npx skills add BerriAI/litellm-skills -g` + +## Troubleshooting + +- `refused to start: no master key is set` means the loader line did not run in the shell command that started `litellm`, or `--env-file .env` is missing from `docker run` +- `a database URL is set but the prisma CLI is neither on PATH nor importable` means the install lacks `extra_proxy`, so rerun step 4 with it, or, on plain routing, that the command that started `litellm` did not start with the loader line +- `The Client hasn't been generated yet` or `Unable to find Prisma binaries` means the Prisma generate command in step 6 has not run +- `Incorrect API key provided` or another provider `AuthenticationError` means a provider key is wrong. Fix it in `.env` and restart, since the proxy reads the environment only at startup. On uv, stop it and rerun step 7. On Docker, `docker rm -f litellm` and rerun step 7, since `docker restart` keeps the environment the container was created with. On the quickstart the key is saved with the model, so delete the model and add it again: `GET /model/info` lists each model's `model_info.id`, and `POST /model/delete` with `{"id": ""}` deletes it +- `Cannot automatically baseline this database` means `DATABASE_URL` points at a database that already holds tables LiteLLM did not create. Point it at a new, empty database +- `LLM Provider NOT provided` means a `model` in `config.yaml` (or step 3) lacks its provider in front. Use the name as the step 5 lookup prints it +- `Invalid model name passed in model=...` means the request's `model` is not a `model_name` in `config.yaml` or step 3. `GET /v1/models` lists the ones the key can use +- `No connected db.` on a proxy without Postgres means the bearer token is not the master key +- `Conflict. The container name "/litellm" is already in use` means an earlier run left a container. `docker start litellm` reuses it, or with the user's OK `docker rm -f litellm` and rerun step 7 +- Inside Docker, `localhost` in `config.yaml` (an Ollama `api_base`, say) is the container itself. Use `host.docker.internal`, and on Linux also pass `--add-host=host.docker.internal:host-gateway` diff --git a/tests/unit/skills/test_skills_data/slack-gif-creator/SKILL.md b/tests/unit/skills/test_skills_data/slack-gif-creator/SKILL.md index 3cae971b731..d2cdfd17bde 100644 --- a/tests/unit/skills/test_skills_data/slack-gif-creator/SKILL.md +++ b/tests/unit/skills/test_skills_data/slack-gif-creator/SKILL.md @@ -1,6 +1,8 @@ --- name: slack-gif-creator description: Knowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." +metadata: + internal: true --- # Slack GIF Creator diff --git a/tests/unit/skills/test_skills_data/test-delete-skill/SKILL.md b/tests/unit/skills/test_skills_data/test-delete-skill/SKILL.md index ba3fda9ab7b..ef1fd1a7b24 100644 --- a/tests/unit/skills/test_skills_data/test-delete-skill/SKILL.md +++ b/tests/unit/skills/test_skills_data/test-delete-skill/SKILL.md @@ -1,6 +1,8 @@ --- name: test-delete-skill description: A test skill created specifically for deletion testing +metadata: + internal: true --- # Test Delete Skill diff --git a/tests/unit/skills/test_skills_data/test-skill-litellm/SKILL.md b/tests/unit/skills/test_skills_data/test-skill-litellm/SKILL.md index ec83b580f0e..b3d54ccdf3c 100644 --- a/tests/unit/skills/test_skills_data/test-skill-litellm/SKILL.md +++ b/tests/unit/skills/test_skills_data/test-skill-litellm/SKILL.md @@ -1,6 +1,8 @@ --- name: test-skill-litellm description: A test skill created by LiteLLM automated tests +metadata: + internal: true --- # Test Skill diff --git a/tests/unit/test_repo_agent_skills.py b/tests/unit/test_repo_agent_skills.py new file mode 100644 index 00000000000..591241f0cf9 --- /dev/null +++ b/tests/unit/test_repo_agent_skills.py @@ -0,0 +1,68 @@ +from collections.abc import Iterator +from fnmatch import fnmatch +from pathlib import Path +from typing import Final + +import pytest +import yaml + +_REPO_ROOT: Final = Path(__file__).resolve().parents[2] +_SKILLS_DIR: Final = _REPO_ROOT / "skills" +_SKILL_DIRECTORIES: Final = tuple(sorted(skill_file.parent for skill_file in _SKILLS_DIR.glob("*/SKILL.md"))) +_FULL_DEPTH_MAX_DEPTH: Final = 5 +_FULL_DEPTH_SKIPPED_NAMES: Final = frozenset(("node_modules", ".git", "dist", "build", "__pycache__")) +_GITIGNORED_NAME_PATTERNS: Final = tuple( + line.strip().rstrip("/") + for line in (_REPO_ROOT / ".gitignore").read_text(encoding="utf-8").splitlines() + if line.strip() and not line.startswith(("#", "!")) and "/" not in line.strip().rstrip("/") +) + + +def _absent_from_a_clone_or_skipped_by_the_cli(name: str) -> bool: + return name in _FULL_DEPTH_SKIPPED_NAMES or any(fnmatch(name, pattern) for pattern in _GITIGNORED_NAME_PATTERNS) + + +def _full_depth_skill_directories(directory: Path, depth: int) -> Iterator[Path]: + if (directory / "SKILL.md").is_file(): + yield directory + if depth == _FULL_DEPTH_MAX_DEPTH: + return + for child in sorted(directory.iterdir()): + if child.is_dir() and not _absent_from_a_clone_or_skipped_by_the_cli(child.name): + yield from _full_depth_skill_directories(child, depth + 1) + + +_NON_END_USER_SKILL_DIRECTORIES: Final = tuple( + directory for directory in _full_depth_skill_directories(_REPO_ROOT, 0) if directory.parent != _SKILLS_DIR +) + + +def _frontmatter(skill_directory: Path) -> object: + opening, frontmatter, _body = (skill_directory / "SKILL.md").read_text(encoding="utf-8").split("---\n", 2) + assert opening == "", f"{skill_directory.name}/SKILL.md must open with a --- frontmatter block" + return yaml.safe_load(frontmatter) + + +@pytest.mark.parametrize("skill_directory", _SKILL_DIRECTORIES, ids=lambda path: path.name) +def test_skill_frontmatter_has_string_name_matching_its_directory_and_a_description(skill_directory: Path) -> None: + frontmatter: Final = _frontmatter(skill_directory) + assert isinstance(frontmatter, dict), frontmatter + assert frontmatter.get("name") == skill_directory.name, frontmatter + description: Final = frontmatter.get("description") + assert isinstance(description, str) and description.strip(), frontmatter + + +@pytest.mark.parametrize( + "skill_directory", + _NON_END_USER_SKILL_DIRECTORIES, + ids=lambda path: str(path.relative_to(_REPO_ROOT)), +) +def test_skill_outside_skills_folder_is_hidden_from_end_user_skill_installs(skill_directory: Path) -> None: + frontmatter: Final = _frontmatter(skill_directory) + assert isinstance(frontmatter, dict), frontmatter + metadata: Final = frontmatter.get("metadata") + assert isinstance(metadata, dict) and metadata.get("internal") is True, ( + f"{skill_directory} needs `metadata: {{internal: true}}`, or `npx skills add BerriAI/litellm` installs it " + "for end users, from a root dot-folder always and from anywhere with --full-depth (skills CLI 1.7.4, " + "checked 2026-10-28)" + ) diff --git a/ui/litellm-dashboard/.agents/skills/url-state/SKILL.md b/ui/litellm-dashboard/.agents/skills/url-state/SKILL.md index 986118d81a7..8bf8fadd19e 100644 --- a/ui/litellm-dashboard/.agents/skills/url-state/SKILL.md +++ b/ui/litellm-dashboard/.agents/skills/url-state/SKILL.md @@ -1,6 +1,8 @@ --- name: url-state description: Read or write dashboard URL query state (tabs, filters, pagination, deep links, demo flags) with nuqs instead of raw history or URLSearchParams +metadata: + internal: true --- # URL state From 752b5c053a50f1cb56d1d099f2660a4459dafc1c Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 28 Oct 2026 15:52:15 -0700 Subject: [PATCH 14/27] fix(ci): flag any schema.prisma change inside model LiteLLM_SpendLogs in the spend logs guard (LIT-10240) (#48905) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ci_cd/spend_logs_guard.py | 107 ++++++++++++++---- tests/unit/test_spend_logs_guard.py | 163 +++++++++++++++++++++++----- 2 files changed, 220 insertions(+), 50 deletions(-) diff --git a/ci_cd/spend_logs_guard.py b/ci_cd/spend_logs_guard.py index a4d9a2e0a0d..5e6d38fba57 100644 --- a/ci_cd/spend_logs_guard.py +++ b/ci_cd/spend_logs_guard.py @@ -2,11 +2,12 @@ needs an approving review from the spend-log owner before it can merge. CODEOWNERS can only match file paths, and every schema change edits the same schema.prisma copies, -so a path rule would put every migration on the owner. This gate reads the diff instead: when no -added line in the watched files mentions the table it passes at once, otherwise it passes only once -the owner's latest review on the pull request is an approval that still covers every spend-log line -in the head (approvals survive pushes, so a later commit may not add spend-log lines the owner never -saw). Reviews do not trigger the workflow, so the check is re-run by hand after approving. +so a path rule would put every migration on the owner. This gate reads the diff instead: a schema.prisma +hunk counts when it adds or removes a line inside the `model LiteLLM_SpendLogs { ... }` block, a migration +hunk counts when an added line names the table. Without such a hunk it passes at once, otherwise it passes +only once the owner's latest review on the pull request is an approval that still covers every spend-log line +in the head (approvals survive pushes, so a later commit may not add spend-log lines the owner never saw). +Reviews do not trigger the workflow, so the check is re-run by hand after approving. """ from __future__ import annotations @@ -15,8 +16,10 @@ import argparse import re import subprocess import sys -from collections.abc import Callable, Sequence +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass from pathlib import Path +from types import MappingProxyType from typing import Final from pydantic import BaseModel, ConfigDict, TypeAdapter @@ -29,6 +32,8 @@ WATCHED: Final = ( MIGRATIONS_PREFIX: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations/" TABLE: Final = re.compile(r"(? bool: return path in WATCHED or (path.startswith(MIGRATIONS_PREFIX) and path.endswith("/migration.sql")) -def _added_lines(section: str) -> tuple[str, ...]: - body: Final = section.split("\n@@", 1) - return tuple(line[1:] for line in body[1].splitlines() if line.startswith("+")) if len(body) == 2 else () +def model_block(schema: str) -> range: + """1-based line range of `model LiteLLM_SpendLogs { ... }` including both braces, empty when the schema has none.""" + lines: Final = schema.splitlines() + starts: Final = tuple(number for number, line in enumerate(lines, 1) if MODEL_START.match(line)) + if not starts: + return range(0) + ends: Final = tuple(number for number, line in enumerate(lines, 1) if number > starts[0] and line.startswith("}")) + return range(starts[0], (ends[0] if ends else len(lines)) + 1) -def spend_log_hits(diff: str) -> tuple[str, ...]: - """`path: added line` for every added line in a watched file that names the table.""" +def _hunks(section: str) -> tuple[Hunk, ...]: + headers: Final = tuple(HUNK.finditer(section)) + ends: Final = tuple(header.start() for header in headers[1:]) + (len(section),) + return tuple(_hunk(header, section[header.end() : end]) for header, end in zip(headers, ends)) + + +def _hunk(header: re.Match[str], body: str) -> Hunk: + lines: Final = body.splitlines() + return Hunk( + removed=tuple(enumerate((line[1:] for line in lines if line.startswith("-")), int(header.group(1)))), + added=tuple(enumerate((line[1:] for line in lines if line.startswith("+")), int(header.group(2)))), + ) + + +def spend_log_hits(diff: str, blocks: Mapping[str, Block]) -> tuple[str, ...]: + """`path: +line` or `path: -line` for every changed line that touches the table. + + A schema.prisma line counts when its number falls inside that copy's `blocks` entry (head for added lines, + merge base for removed ones), a migration line when it is added and names the table.""" sections: Final = re.split(r"^diff --git [^\n]*\n", diff, flags=re.MULTILINE) - return tuple(f"{path}: {line.strip()}" for path, line in _watched_added_lines(sections) if TABLE.search(line)) - - -def _watched_added_lines(sections: Sequence[str]) -> tuple[tuple[str, str], ...]: - pairs: Final = tuple((_path_of(section), section) for section in sections) + paths: Final = tuple((_path_of(section), section) for section in sections) return tuple( - (path, line) for path, section in pairs if watched(path) for line in _added_lines(section) - ) # comprehension-ok: path then its lines + hit for path, section in paths if watched(path) for hit in _section_hits(path, _hunks(section), blocks) + ) # comprehension-ok: path then its hits + + +def _section_hits(path: str, hunks: Sequence[Hunk], blocks: Mapping[str, Block]) -> tuple[str, ...]: + if path not in blocks: + return tuple(f"{path}: +{line.strip()}" for hunk in hunks for _, line in hunk.added if TABLE.search(line)) + block: Final = blocks[path] + removed: Final = tuple( + f"{path}: -{line.strip()}" for hunk in hunks for number, line in hunk.removed if number in block.base + ) # comprehension-ok: hunk then its lines + added: Final = tuple( + f"{path}: +{line.strip()}" for hunk in hunks for number, line in hunk.added if number in block.head + ) # comprehension-ok: hunk then its lines + return removed + added def _path_of(section: str) -> str: @@ -111,13 +161,25 @@ def added_diff(base: str, head: str) -> str: return result.stdout +def file_at(revision: str, path: str) -> str: + """Contents of `path` at `revision`, empty when the file does not exist there.""" + result: Final = subprocess.run(("git", "show", f"{revision}:{path}"), check=False, capture_output=True, text=True) + return result.stdout if result.returncode == 0 else "" + + +def schema_blocks(base: str, head: str) -> Mapping[str, Block]: + return MappingProxyType( + {path: Block(base=model_block(file_at(base, path)), head=model_block(file_at(head, path))) for path in WATCHED} + ) + + def fetched(commit: str) -> bool: result: Final = subprocess.run(("git", "fetch", "--no-tags", "--depth=1", "origin", commit), check=False) return result.returncode == 0 def reviewed_hits(base: str, commit: str) -> tuple[str, ...] | None: - return spend_log_hits(added_diff(base, commit)) if fetched(commit) else None + return spend_log_hits(added_diff(base, commit), schema_blocks(base, commit)) if fetched(commit) else None class Arguments(BaseModel): @@ -135,9 +197,12 @@ def main(argv: Sequence[str]) -> int: parser.add_argument("--approver", required=True, help="GitHub login whose approval unlocks the merge") parser.add_argument("--reviews", required=True, help="JSON file with the pull request's reviews") args: Final = Arguments.model_validate(vars(parser.parse_args(argv))) - hits: Final = spend_log_hits(added_diff(args.base, args.head)) + hits: Final = spend_log_hits(added_diff(args.base, args.head), schema_blocks(args.base, args.head)) if not hits: - sys.stdout.write("No added line in schema.prisma or a migration names LiteLLM_SpendLogs; nothing to review.\n") + sys.stdout.write( + "No change inside model LiteLLM_SpendLogs in schema.prisma and no added migration line names it; " + "nothing to review.\n" + ) return 0 reviews: Final = _REVIEWS.validate_json(args.reviews.read_bytes()) approval: Final = latest_approval(reviews, args.approver) if approved_by(reviews, args.approver) else None diff --git a/tests/unit/test_spend_logs_guard.py b/tests/unit/test_spend_logs_guard.py index 9ac2c923632..eb54099cf8b 100644 --- a/tests/unit/test_spend_logs_guard.py +++ b/tests/unit/test_spend_logs_guard.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import importlib.util import sys from pathlib import Path @@ -12,46 +14,144 @@ guard: Final = importlib.util.module_from_spec(_SPEC) sys.modules["spend_logs_guard"] = guard _SPEC.loader.exec_module(guard) +SCHEMA: Final = ( + "generator client {\n" + ' provider = "prisma-client-py"\n' + "}\n" + "\n" + "model LiteLLM_TeamTable {\n" + " team_id String @id\n" + "}\n" + "\n" + "model LiteLLM_SpendLogs {\n" + " request_id String @id\n" + " api_key String\n" + ' @@index([api_key], map: "LiteLLM_SpendLogs_api_key_idx")\n' + "}\n" + "\n" + "model LiteLLM_UserTable {\n" + " user_id String @id\n" + "}\n" +) +_NO_BLOCKS: Final = {} -def _diff(path: str, added: str, start: int = 10) -> str: - return f"diff --git a/{path} b/{path}\n--- a/{path}\n+++ b/{path}\n@@ -{start},0 +{start},1 @@\n+{added}\n" + +def _hunk(old_start: int, old_count: int, new_start: int, new_count: int, *lines: str) -> str: + body: Final = "".join(f"{line}\n" for line in lines) + return f"@@ -{old_start},{old_count} +{new_start},{new_count} @@\n{body}" + + +def _diff(path: str, *hunks: str) -> str: + return f"diff --git a/{path} b/{path}\n--- a/{path}\n+++ b/{path}\n" + "".join(hunks) + + +def _added(path: str, added: str, start: int = 10) -> str: + return _diff(path, _hunk(start, 0, start, 1, f"+{added}")) + + +def _blocks(path: str, base: str = SCHEMA, head: str = SCHEMA) -> dict[str, guard.Block]: + return {path: guard.Block(base=guard.model_block(base), head=guard.model_block(head))} + + +def _with_line(schema: str, after: int, line: str) -> str: + lines: Final = schema.splitlines(keepends=True) + return "".join(lines[:after]) + f"{line}\n" + "".join(lines[after:]) + + +class TestModelBlock: + def test_the_block_runs_from_the_model_line_through_its_closing_brace(self): + assert guard.model_block(SCHEMA) == range(9, 14) + + def test_a_schema_without_the_model_has_an_empty_block(self): + assert guard.model_block(SCHEMA.replace("LiteLLM_SpendLogs", "LiteLLM_ErrorLogs")) == range(0) class TestSpendLogHits: - def test_an_added_schema_line_naming_the_table_is_a_hit(self): + def test_a_field_added_inside_the_block_is_a_hit_without_naming_the_table(self): + head: Final = _with_line(SCHEMA, 9, " gauntlet_probe String?") + diff: Final = _added("schema.prisma", " gauntlet_probe String?") + assert guard.spend_log_hits(diff, _blocks("schema.prisma", head=head)) == ( + "schema.prisma: +gauntlet_probe String?", + ) + + def test_a_field_removed_from_the_block_is_a_hit(self): + head: Final = SCHEMA.replace(" api_key String\n", "") + diff: Final = _diff("schema.prisma", _hunk(11, 1, 10, 0, "- api_key String")) + assert guard.spend_log_hits(diff, _blocks("schema.prisma", head=head)) == ("schema.prisma: -api_key String",) + + def test_the_whole_model_appearing_or_disappearing_is_a_hit(self): + without: Final = SCHEMA.replace( + SCHEMA[SCHEMA.index("model LiteLLM_SpendLogs") : SCHEMA.index("model LiteLLM_UserTable")], "" + ) + body: Final = tuple(SCHEMA.splitlines()[8:14]) + added: Final = _diff("schema.prisma", _hunk(8, 0, 9, 6, *(f"+{line}" for line in body))) + removed: Final = _diff("schema.prisma", _hunk(9, 6, 8, 0, *(f"-{line}" for line in body))) + assert len(guard.spend_log_hits(added, _blocks("schema.prisma", base=without))) == 5 + assert len(guard.spend_log_hits(removed, _blocks("schema.prisma", head=without))) == 5 + + def test_a_line_naming_the_table_in_another_model_is_not_a_hit(self): + head: Final = _with_line(SCHEMA, 6, " spend_logs LiteLLM_SpendLogs[]") diff: Final = _diff( - "schema.prisma", ' @@index([api_key, startTime], map: "LiteLLM_SpendLogs_api_key_startTime_idx")', 740 - ) - assert guard.spend_log_hits(diff) == ( - 'schema.prisma: @@index([api_key, startTime], map: "LiteLLM_SpendLogs_api_key_startTime_idx")', + "schema.prisma", + _hunk(6, 0, 7, 1, "+ spend_logs LiteLLM_SpendLogs[]"), + _hunk(16, 1, 17, 0, "- user_id String @id"), ) + assert guard.spend_log_hits(diff, _blocks("schema.prisma", head=head)) == () - def test_an_added_migration_line_is_a_hit_in_any_case(self): + def test_each_hunk_is_placed_by_its_own_header(self): + head: Final = _with_line(_with_line(SCHEMA, 2, ' output = "./client"'), 10, " new_col String?") + diff: Final = _diff( + "schema.prisma", + _hunk(2, 0, 3, 1, '+ output = "./client"'), + _hunk(9, 0, 11, 1, "+ new_col String?"), + ) + assert guard.spend_log_hits(diff, _blocks("schema.prisma", head=head)) == ("schema.prisma: +new_col String?",) + + def test_an_added_migration_line_is_a_hit_in_any_case_and_a_removed_one_is_not(self): path: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations/20260101000000_x/migration.sql" - hits: Final = guard.spend_log_hits(_diff(path, 'CREATE INDEX "i" ON "litellm_spendlogs"("api_key");', 3)) - assert hits == (f'{path}: CREATE INDEX "i" ON "litellm_spendlogs"("api_key");',) + diff: Final = _diff( + path, + _hunk( + 3, + 1, + 3, + 1, + '-DROP INDEX "LiteLLM_SpendLogs_old_idx";', + '+CREATE INDEX "i" ON "litellm_spendlogs"("api_key");', + ), + ) + assert guard.spend_log_hits(diff, _NO_BLOCKS) == ( + f'{path}: +CREATE INDEX "i" ON "litellm_spendlogs"("api_key");', + ) - def test_a_removed_line_or_another_table_or_an_unwatched_file_is_not_a_hit(self): - removed: Final = "--- a/schema.prisma\n+++ b/schema.prisma\n@@ -1,1 +1,0 @@\n-model LiteLLM_SpendLogs {\n" - other: Final = _diff("schema.prisma", " team_id String // LiteLLM_TeamTable, not MyLiteLLM_SpendLogs", 5) - elsewhere: Final = _diff("litellm/proxy/utils.py", 'table = "LiteLLM_SpendLogs"', 5) - assert guard.spend_log_hits(removed + other + elsewhere) == () + def test_another_table_in_a_migration_or_an_unwatched_file_is_not_a_hit(self): + path: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations/20260101000000_x/migration.sql" + other: Final = _added( + path, 'ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN "x" TEXT; -- not MyLiteLLM_SpendLogs', 5 + ) + elsewhere: Final = _added("litellm/proxy/utils.py", 'table = "LiteLLM_SpendLogs"', 5) + assert guard.spend_log_hits(other + elsewhere, _NO_BLOCKS) == () def test_a_table_whose_name_only_starts_with_the_watched_one_is_not_a_hit(self): - archive: Final = _diff("schema.prisma", "model LiteLLM_SpendLogsArchive {", 900) - index: Final = _diff("schema.prisma", " @@index([team_id], map: \"LiteLLM_SpendLogs_team_id_idx\")", 760) - assert guard.spend_log_hits(archive + index) == ( - 'schema.prisma: @@index([team_id], map: "LiteLLM_SpendLogs_team_id_idx")', + index: Final = ' @@index([team_id], map: "LiteLLM_SpendLogs_team_id_idx")' + head: Final = _with_line(_with_line(SCHEMA, 18, "model LiteLLM_SpendLogsArchive {"), 12, index) + diff: Final = _diff( + "schema.prisma", + _hunk(12, 0, 13, 1, f"+{index}"), + _hunk(18, 0, 20, 1, "+model LiteLLM_SpendLogsArchive {"), + ) + assert guard.spend_log_hits(diff, _blocks("schema.prisma", head=head)) == ( + 'schema.prisma: +@@index([team_id], map: "LiteLLM_SpendLogs_team_id_idx")', ) def test_two_files_in_one_diff_are_attributed_to_their_own_path(self): - diff: Final = _diff("litellm/proxy/schema.prisma", " a LiteLLM_SpendLogs", 1) + _diff( - "schema.prisma", " b String", 1 - ) - assert guard.spend_log_hits(diff) == ("litellm/proxy/schema.prisma: a LiteLLM_SpendLogs",) + head: Final = _with_line(SCHEMA, 9, " a String") + diff: Final = _added("litellm/proxy/schema.prisma", " a String") + _added("schema.prisma", " b String", 2) + blocks: Final = _blocks("litellm/proxy/schema.prisma", head=head) | _blocks("schema.prisma") + assert guard.spend_log_hits(diff, blocks) == ("litellm/proxy/schema.prisma: +a String",) -def _review(login: str, state: str, commit_id: str | None = "head") -> "guard.Review": +def _review(login: str, state: str, commit_id: str | None = "head") -> guard.Review: return guard.Review(state=state, user=guard.Reviewer(login=login), commit_id=commit_id) @@ -126,24 +226,29 @@ class TestApprovalCovers: class TestExitCode: def test_a_pull_request_without_spend_log_lines_passes_without_reviews(self, tmp_path, monkeypatch, capsys): - monkeypatch.setattr(guard, "added_diff", lambda base, head: _diff("schema.prisma", " foo String", 1)) + monkeypatch.setattr(guard, "added_diff", lambda base, head: _added("schema.prisma", " foo String", 6)) + monkeypatch.setattr(guard, "schema_blocks", lambda base, head: _blocks("schema.prisma")) reviews: Final = tmp_path / "r.json" reviews.write_text("[]") assert guard.main(["--base", "a", "--head", "b", "--approver", "kerry-berri", "--reviews", str(reviews)]) == 0 assert "nothing to review" in capsys.readouterr().out def test_a_spend_log_change_fails_until_the_approver_approves(self, tmp_path, monkeypatch, capsys): - monkeypatch.setattr(guard, "added_diff", lambda base, head: _diff("schema.prisma", " x LiteLLM_SpendLogs", 7)) + head: Final = _with_line(SCHEMA, 9, " x String?") + monkeypatch.setattr(guard, "added_diff", lambda base_sha, head_sha: _added("schema.prisma", " x String?")) + monkeypatch.setattr(guard, "schema_blocks", lambda base_sha, head_sha: _blocks("schema.prisma", head=head)) reviews: Final = tmp_path / "r.json" reviews.write_text('[{"user": {"login": "someone-else"}, "state": "APPROVED"}]') assert guard.main(["--base", "a", "--head", "b", "--approver", "kerry-berri", "--reviews", str(reviews)]) == 1 - assert "schema.prisma: x LiteLLM_SpendLogs" in capsys.readouterr().out + assert "schema.prisma: +x String?" in capsys.readouterr().out reviews.write_text('[{"user": {"login": "kerry-berri"}, "state": "APPROVED", "commit_id": "b"}]') assert guard.main(["--base", "a", "--head", "b", "--approver", "kerry-berri", "--reviews", str(reviews)]) == 0 def test_an_approval_before_a_new_spend_log_line_was_pushed_fails(self, tmp_path, monkeypatch, capsys): - monkeypatch.setattr(guard, "added_diff", lambda base, head: _diff("schema.prisma", " x LiteLLM_SpendLogs", 7)) - monkeypatch.setattr(guard, "reviewed_hits", lambda base, commit: () if commit == "old" else None, raising=False) + head: Final = _with_line(SCHEMA, 9, " x String?") + monkeypatch.setattr(guard, "added_diff", lambda base_sha, head_sha: _added("schema.prisma", " x String?")) + monkeypatch.setattr(guard, "schema_blocks", lambda base_sha, head_sha: _blocks("schema.prisma", head=head)) + monkeypatch.setattr(guard, "reviewed_hits", lambda base_sha, commit: () if commit == "old" else None) reviews: Final = tmp_path / "r.json" reviews.write_text('[{"user": {"login": "kerry-berri"}, "state": "APPROVED", "commit_id": "old"}]') assert guard.main(["--base", "a", "--head", "b", "--approver", "kerry-berri", "--reviews", str(reviews)]) == 1 From 0f786fa1c7b99ee860643597675ed7daf0d289d0 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 28 Oct 2026 15:52:44 -0700 Subject: [PATCH 15/27] feat(router): add deployment rpd and tpd daily limits to enforce_model_rate_limits (#48875) * feat(router): add deployment rpd and tpd daily limits to enforce_model_rate_limits Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): route rate limit cache increments through typed helpers * chore(ui): regenerate schema.d.ts for deployment rpd and tpd fields * test(types): add rpd and tpd to the owned litellm_params inventory * fix(router): run daily rpd/tpd checks after the minute checks and increment rpm and rpd in one cache trip * fix(router): flatten the async rpm reject branch for the ruff strict gate * fix(router): hand rpm and rpd back on any rate limit reject A request the rpm, rpd or tpd check rejects no longer keeps its request counts, so retries against a deployment whose rpd is spent cannot fill its rpm and block the in-group fallover with the minute 429. * test(router): a minute-rejected request no longer keeps its rpm count --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/in_memory_cache.py | 4 +- .../litellm_core_utils/get_litellm_params.py | 2 + .../pre_call_checks/model_rate_limit_check.py | 228 +++++++++++--- litellm/types/litellm_params.py | 2 + litellm/types/router.py | 4 + ..._enforce_model_rate_limits_daily_limits.py | 96 ++++++ tests/unit/caching/test_in_memory_cache.py | 25 +- .../test_enforce_model_rate_limits.py | 286 +++++++++++++++++- .../test_router/test_io_token_rate_limits.py | 2 +- tests/unit/types/test_litellm_params.py | 2 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 8 + 11 files changed, 589 insertions(+), 70 deletions(-) create mode 100644 tests/integration/routing/test_enforce_model_rate_limits_daily_limits.py diff --git a/litellm/caching/in_memory_cache.py b/litellm/caching/in_memory_cache.py index 1cc3e106aa0..6b40c48c4ef 100644 --- a/litellm/caching/in_memory_cache.py +++ b/litellm/caching/in_memory_cache.py @@ -274,7 +274,9 @@ class InMemoryCache(BaseCache): ) -> list[float] | None: results: Final = [] for increment in increment_list: - result = await self.async_increment(increment["key"], increment["increment_value"], **kwargs) + result = await self.async_increment( + increment["key"], increment["increment_value"], ttl=increment["ttl"], **kwargs + ) results.append(result) return results diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index ba964d200ec..8178b7e98b8 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -75,6 +75,8 @@ OPTIONAL_KWARGS_KEYS: Final = ( "rpm", "itpm", "otpm", + "tpd", + "rpd", "use_xai_oauth", "chatgpt_auth_file", "github_copilot_auth_type", diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index ea3fdd601dc..9eebcf39515 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -5,15 +5,19 @@ When enabled via router_settings.optional_pre_call_checks: ["enforce_model_rate_ - tpm/rpm: combined TPM + optional RPM (legacy) - itpm/otpm: separate input/output tokens per minute +- tpd/rpd: tokens and requests per UTC calendar day When a deployment sets both itpm/otpm and tpm/rpm, both are enforced. A warning is logged the first time such a deployment is seen. """ import contextlib -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Callable +from datetime import datetime, timedelta +from typing import TYPE_CHECKING, Any, Final, Literal import httpx +from pydantic import BaseModel, ConfigDict import litellm from litellm._internal_context import with_service_target @@ -33,6 +37,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( io_token_refund_failure, ) from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET +from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.router import RouterErrors from litellm.types.utils import StandardLoggingPayload from litellm.utils import get_utc_datetime @@ -47,6 +52,62 @@ else: class RoutingArgs: ttl: int = 60 # 1min (RPM/TPM expire key) + daily_ttl: int = 86400 + + +DAY_FORMAT: Final = "%Y-%m-%d" + + +def seconds_until_utc_midnight(now: datetime) -> int: + next_midnight: Final = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) + return max(1, int((next_midnight - now).total_seconds())) + + +def _first_set(*values: int | None) -> int | None: + return next((value for value in values if value is not None), None) + + +class _DailyLimitFields(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) + tpd: int | None = None + rpd: int | None = None + + +class _DailyLimitParams(_DailyLimitFields): + model: str | None = None + + +class _DailyLimitModelInfo(_DailyLimitFields): + id: str | None = None + + +class _DailyLimitDeployment(_DailyLimitFields): + model_name: str = "" + litellm_params: _DailyLimitParams = _DailyLimitParams() + model_info: _DailyLimitModelInfo = _DailyLimitModelInfo() + + def limits(self) -> tuple[int | None, int | None]: + return ( + _first_set(self.tpd, self.litellm_params.tpd, self.model_info.tpd), + _first_set(self.rpd, self.litellm_params.rpd, self.model_info.rpd), + ) + + def cache_keys(self, current_day: str) -> tuple[str, str]: + prefix: Final = f"{self.model_info.id}:{self.litellm_params.model}" + return f"{prefix}:tpd:{current_day}", f"{prefix}:rpd:{current_day}" + + def limit_error(self, kind: Literal["tpd", "rpd"], limit: int, usage: int, now: datetime) -> litellm.RateLimitError: + return litellm.RateLimitError( + message=f"Model rate limit exceeded. {kind.upper()} limit={limit}, current usage={usage}", + llm_provider="", + model=self.litellm_params.model, + response=httpx.Response( + status_code=429, + content=f"{RouterErrors.user_defined_ratelimit_error.value} {kind} limit={limit}. current usage={usage}. id={self.model_info.id}, model_group={self.model_name}", + headers={"retry-after": str(seconds_until_utc_midnight(now))}, + request=httpx.Request(method="model_rate_limit_check", url="https://github.com/BerriAI/litellm"), + ), + ) class ModelRateLimitingCheck(CustomLogger): @@ -60,12 +121,60 @@ class ModelRateLimitingCheck(CustomLogger): this check actively enforces those limits across ALL routing strategies. """ - def __init__(self, dual_cache: DualCache): + def __init__(self, dual_cache: DualCache, clock: Callable[[], datetime] = get_utc_datetime): self.dual_cache = dual_cache + self.clock = clock # model_ids already warned about itpm/otpm + tpm/rpm on the same deployment, # so the warning is logged once per deployment rather than per request. self._io_token_conflict_warned_ids: set[str] = set() + def _increment(self, key: str, value: int, ttl: int) -> int: + return self.dual_cache.increment_cache(key=key, value=value, ttl=ttl) + + async def _async_increment( + self, key: str, value: int, ttl: int, parent_otel_span: Span | None = None + ) -> int | None: + count: Final = await self.dual_cache.async_increment_cache( + key=key, value=value, ttl=ttl, parent_otel_span=parent_otel_span + ) + return None if count is None else int(count) + + @staticmethod + def _request_count_operations( + rpm_key: str | None, rpd_key: str | None, increment_value: int + ) -> tuple[RedisPipelineIncrementOperation, ...]: + return tuple( + RedisPipelineIncrementOperation(key=key, increment_value=increment_value, ttl=ttl) + for key, ttl in ((rpm_key, RoutingArgs.ttl), (rpd_key, RoutingArgs.daily_ttl)) + if key is not None + ) + + async def _async_refund_request_counts( + self, rpm_key: str | None, rpd_key: str | None, parent_otel_span: Span | None + ) -> None: + operations: Final = self._request_count_operations(rpm_key, rpd_key, -1) + if operations: + await self.dual_cache.async_increment_cache_pipeline( + increment_list=list(operations), parent_otel_span=parent_otel_span + ) + + async def _async_increment_request_counts( + self, rpm_key: str | None, rpd_key: str | None, parent_otel_span: Span | None + ) -> tuple[int | None, int | None]: + operations: Final = self._request_count_operations(rpm_key, rpd_key, 1) + if not operations: + return None, None + counts: Final = await self.dual_cache.async_increment_cache_pipeline( + increment_list=list(operations), parent_otel_span=parent_otel_span + ) + if counts is None or len(counts) != len(operations): + return None, None + count_by_key: Final = {op["key"]: int(count) for op, count in zip(operations, counts)} + return ( + None if rpm_key is None else count_by_key.get(rpm_key), + None if rpd_key is None else count_by_key.get(rpd_key), + ) + def _warn_io_token_and_tpm_rpm_coexist_once(self, deployment: dict) -> None: tpm_limit, rpm_limit = self._get_deployment_limits(deployment) if tpm_limit is None and rpm_limit is None: @@ -129,6 +238,21 @@ class ModelRateLimitingCheck(CustomLogger): return tpm, rpm + def _check_daily_limits(self, deployment: _DailyLimitDeployment, now: datetime) -> None: + tpd_limit, rpd_limit = deployment.limits() + if tpd_limit is None and rpd_limit is None: + return + tpd_key, rpd_key = deployment.cache_keys(now.strftime(DAY_FORMAT)) + if tpd_limit is not None: + current_tpd: Final = self._get_current_tpm(tpd_key, tpd_limit) + if current_tpd is not None and current_tpd >= tpd_limit: + raise deployment.limit_error("tpd", tpd_limit, current_tpd, now) + if rpd_limit is not None: + current_rpd: Final = self._increment(rpd_key, 1, RoutingArgs.daily_ttl) + if current_rpd > rpd_limit: + self._increment(rpd_key, -1, RoutingArgs.daily_ttl) + raise deployment.limit_error("rpd", rpd_limit, current_rpd, now) + def _get_cache_keys(self, deployment: dict, current_minute: str) -> tuple[str, str]: """Get the cache keys for TPM and RPM tracking.""" model_id: Final = deployment.get("model_info", {}).get("id") @@ -178,13 +302,10 @@ class ModelRateLimitingCheck(CustomLogger): ) io_reservation_made = True + dt: Final = self.clock() + daily: Final = _DailyLimitDeployment.model_validate(deployment) tpm_limit, rpm_limit = self._get_deployment_limits(deployment) - # If no limits are set, allow the request - if tpm_limit is None and rpm_limit is None: - return deployment - - dt: Final = get_utc_datetime() current_minute: Final = dt.strftime("%H-%M") tpm_key, rpm_key = self._get_cache_keys(deployment, current_minute) @@ -213,8 +334,9 @@ class ModelRateLimitingCheck(CustomLogger): # Check RPM limit (atomic increment-first to avoid race conditions) if rpm_limit is not None: - current_rpm: Final = self.dual_cache.increment_cache(key=rpm_key, value=1, ttl=RoutingArgs.ttl) + current_rpm: Final = self._increment(rpm_key, 1, RoutingArgs.ttl) if current_rpm is not None and current_rpm > rpm_limit: + self._increment(rpm_key, -1, RoutingArgs.ttl) raise litellm.RateLimitError( message=f"Model rate limit exceeded. RPM limit={rpm_limit}, current usage={current_rpm}", llm_provider="", @@ -230,6 +352,12 @@ class ModelRateLimitingCheck(CustomLogger): ), ) + try: + self._check_daily_limits(daily, dt) + except litellm.RateLimitError: + if rpm_limit is not None: + self._increment(rpm_key, -1, RoutingArgs.ttl) + raise return deployment except litellm.RateLimitError: @@ -259,15 +387,14 @@ class ModelRateLimitingCheck(CustomLogger): ) io_reservation_made = True + dt: Final = self.clock() + daily: Final = _DailyLimitDeployment.model_validate(deployment) + tpd_limit, rpd_limit = daily.limits() tpm_limit, rpm_limit = self._get_deployment_limits(deployment) - # If no limits are set, allow the request - if tpm_limit is None and rpm_limit is None: - return deployment - - dt: Final = get_utc_datetime() current_minute: Final = dt.strftime("%H-%M") tpm_key, rpm_key = self._get_cache_keys(deployment, current_minute) + tpd_key, rpd_key = daily.cache_keys(dt.strftime(DAY_FORMAT)) model_id: Final = deployment.get("model_info", {}).get("id") model_name: Final = deployment.get("litellm_params", {}).get("model") @@ -293,30 +420,37 @@ class ModelRateLimitingCheck(CustomLogger): num_retries=0, # Don't retry - return 429 immediately ) - # Check RPM limit (atomic increment-first to avoid race conditions) - if rpm_limit is not None: - current_rpm: Final = await self.dual_cache.async_increment_cache( - key=rpm_key, - value=1, - ttl=RoutingArgs.ttl, - parent_otel_span=parent_otel_span, - ) - if current_rpm is not None and current_rpm > rpm_limit: - raise litellm.RateLimitError( - message=f"Model rate limit exceeded. RPM limit={rpm_limit}, current usage={current_rpm}", - llm_provider="", - model=model_name, - response=httpx.Response( - status_code=429, - content=f"{RouterErrors.user_defined_ratelimit_error.value} rpm limit={rpm_limit}. current usage={current_rpm}. id={model_id}, model_group={model_group}", - headers={"retry-after": str(60)}, - request=httpx.Request( - method="model_rate_limit_check", - url="https://github.com/BerriAI/litellm", - ), + if tpd_limit is not None: + current_tpd: Final = await self._async_get_current_tpm(tpd_key, tpd_limit, parent_otel_span) + if current_tpd is not None and current_tpd >= tpd_limit: + raise daily.limit_error("tpd", tpd_limit, current_tpd, dt) + + charged_rpm_key: Final = rpm_key if rpm_limit is not None else None + charged_rpd_key: Final = rpd_key if rpd_limit is not None else None + current_rpm, current_rpd = await self._async_increment_request_counts( + charged_rpm_key, charged_rpd_key, parent_otel_span + ) + if rpm_limit is not None and current_rpm is not None and current_rpm > rpm_limit: + await self._async_refund_request_counts(charged_rpm_key, charged_rpd_key, parent_otel_span) + raise litellm.RateLimitError( + message=f"Model rate limit exceeded. RPM limit={rpm_limit}, current usage={current_rpm}", + llm_provider="", + model=model_name, + response=httpx.Response( + status_code=429, + content=f"{RouterErrors.user_defined_ratelimit_error.value} rpm limit={rpm_limit}. current usage={current_rpm}. id={model_id}, model_group={model_group}", + headers={"retry-after": str(60)}, + request=httpx.Request( + method="model_rate_limit_check", + url="https://github.com/BerriAI/litellm", ), - num_retries=0, # Don't retry - return 429 immediately - ) + ), + num_retries=0, # Don't retry - return 429 immediately + ) + + if rpd_limit is not None and current_rpd is not None and current_rpd > rpd_limit: + await self._async_refund_request_counts(charged_rpm_key, charged_rpd_key, parent_otel_span) + raise daily.limit_error("rpd", rpd_limit, current_rpd, dt) return deployment @@ -373,17 +507,16 @@ class ModelRateLimitingCheck(CustomLogger): if not model or not total_tokens: return - dt: Final = get_utc_datetime() + dt: Final = self.clock() current_minute: Final = dt.strftime("%H-%M") tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}" + tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}" verbose_router_logger.debug("[TPM TRACKING] Incrementing %s by %s", tpm_key, total_tokens) - await self.dual_cache.async_increment_cache( - key=tpm_key, - value=total_tokens, - ttl=RoutingArgs.ttl, - ) + tokens: Final = int(total_tokens) + await self._async_increment(tpm_key, tokens, RoutingArgs.ttl) + await self._async_increment(tpd_key, tokens, RoutingArgs.daily_ttl) except Exception as e: verbose_router_logger.debug("Error in ModelRateLimitingCheck.async_log_success_event: %s", e) @@ -434,15 +567,14 @@ class ModelRateLimitingCheck(CustomLogger): if not model or not total_tokens: return - dt: Final = get_utc_datetime() + dt: Final = self.clock() current_minute: Final = dt.strftime("%H-%M") tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}" + tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}" - self.dual_cache.increment_cache( - key=tpm_key, - value=total_tokens, - ttl=RoutingArgs.ttl, - ) + tokens: Final = int(total_tokens) + self._increment(tpm_key, tokens, RoutingArgs.ttl) + self._increment(tpd_key, tokens, RoutingArgs.daily_ttl) except Exception as e: verbose_router_logger.debug("Error in ModelRateLimitingCheck.log_success_event: %s", e) diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 717b74e5741..9196b7a226f 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -189,6 +189,8 @@ class DeploymentOptions: tpm: int | None = None itpm: int | None = None otpm: int | None = None + rpd: int | None = None + tpd: int | None = None kubernetes_pod_discovery: bool | None = None default_api_key_rpm_limit: int | None = None default_api_key_tpm_limit: int | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index ba9b82e2a57..f8de2128a80 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -533,6 +533,8 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): rpm: int | None = None itpm: int | None = None otpm: int | None = None + tpd: int | None = None + rpd: int | None = None timeout: float | str | httpx.Timeout | SDKTimeout | None = None # if str, pass in as os.environ/ # read timeout of a live stream, a faked stream takes the longer of this and timeout. If str, pass in as os.environ/ stream_timeout: float | str | None = None @@ -738,6 +740,8 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): rpm: int | None itpm: int | None otpm: int | None + tpd: ReadOnly[int | None] + rpd: ReadOnly[int | None] order: int | None weight: int | None max_parallel_requests: int | None diff --git a/tests/integration/routing/test_enforce_model_rate_limits_daily_limits.py b/tests/integration/routing/test_enforce_model_rate_limits_daily_limits.py new file mode 100644 index 00000000000..c670171a149 --- /dev/null +++ b/tests/integration/routing/test_enforce_model_rate_limits_daily_limits.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import uuid +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import httpx +import pytest +from integration._support.client import gateway_from_environment +from integration._support.openai_wire import chat_reply +from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from integration.providers._cache_control_marks_support import owned_config +from integration.routing.test_payment_required_mapping import OPENAI_MODEL, is_model_info_probe, model_list_reply +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +GROUP: Final = "daily-limited-group" +LIMITED_ID: Final = "free-tier-two-a-day" +UNLIMITED_ID: Final = "paid-tier" +RPD: Final = 2 +EXTRA_REQUESTS: Final = 2 + + +@dataclass(frozen=True, slots=True) +class Rig: + owned: OwnedProxy + limited: Wire + unlimited: Wire + + +def _reply(request: Request) -> Reply: + if is_model_info_probe(request): + return model_list_reply() + return chat_reply("chatcmpl-daily", OPENAI_MODEL, "served", stream=False) + + +def _deployment(model_id: str, wire: Wire, **params: JsonValue) -> dict[str, JsonValue]: + return { + "model_name": GROUP, + "litellm_params": { + "model": OPENAI_MODEL, + "api_base": wire.url + "/v1", + "api_key": "synthetic-openai-key", + **params, + }, + "model_info": {"id": model_id}, + } + + +def _config(directory: Path, limited: Wire, unlimited: Wire) -> Path: + return owned_config( + directory, + [_deployment(LIMITED_ID, limited, rpd=RPD, order=1), _deployment(UNLIMITED_ID, unlimited, order=2)], + litellm_settings={"cache": False}, + router_settings={"optional_pre_call_checks": ["enforce_model_rate_limits"], "num_retries": 1}, + ) + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("daily-limits") + with ( + gateway_from_environment() as gateway, + wire_server(_reply) as limited, + wire_server(_reply) as unlimited, + owned_proxy_process(gateway, directory, {}, config=_config(directory, limited, unlimited)) as owned, + ): + yield Rig(owned, limited, unlimited) + + +def _chat(rig: Rig) -> httpx.Response: + return rig.owned.gateway.request( + "POST", + "/v1/chat/completions", + {"model": GROUP, "messages": [{"role": "user", "content": f"daily {uuid.uuid4().hex}"}]}, + ) + + +def _forwarded(wire: Wire) -> int: + return sum(1 for request in wire.drain() if not is_model_info_probe(request)) + + +def test_requests_past_the_deployment_rpd_are_served_by_the_other_deployment_in_the_group(rig: Rig) -> None: + responses: Final = tuple(_chat(rig) for _ in range(RPD + EXTRA_REQUESTS)) + + assert [response.status_code for response in responses] == [200] * (RPD + EXTRA_REQUESTS), [ + response.text for response in responses + ] + assert [response.headers["x-litellm-model-id"] for response in responses] == [LIMITED_ID] * RPD + [ + UNLIMITED_ID + ] * EXTRA_REQUESTS + assert (_forwarded(rig.limited), _forwarded(rig.unlimited)) == (RPD, EXTRA_REQUESTS) diff --git a/tests/unit/caching/test_in_memory_cache.py b/tests/unit/caching/test_in_memory_cache.py index 75232476ecd..901832937b5 100644 --- a/tests/unit/caching/test_in_memory_cache.py +++ b/tests/unit/caching/test_in_memory_cache.py @@ -235,13 +235,9 @@ def test_in_memory_cache_eviction_order(): # Add items with different TTLs now = time.time() - in_memory_cache.set_cache( - key="early_expire", value="value_1", ttl=100 - ) # expires in 100 seconds + in_memory_cache.set_cache(key="early_expire", value="value_1", ttl=100) # expires in 100 seconds time.sleep(0.01) - in_memory_cache.set_cache( - key="late_expire", value="value_2", ttl=200 - ) # expires in 200 seconds + in_memory_cache.set_cache(key="late_expire", value="value_2", ttl=200) # expires in 200 seconds # Verify TTL order early_ttl = in_memory_cache.ttl_dict["early_expire"] @@ -379,3 +375,20 @@ def test_in_memory_cache_injected_clock_controls_expiry_and_eviction() -> None: assert cache.get_cache("second") is None assert cache.get_cache("third") == "replacement" assert cache.get_cache("fourth") == "new" + + +@pytest.mark.asyncio +async def test_async_increment_pipeline_keeps_each_key_for_its_own_ttl() -> None: + now: Final = {"t": 1_000.0} + cache: Final = InMemoryCache(default_ttl=600, clock=lambda: now["t"]) + + await cache.async_increment_pipeline( + increment_list=[ + {"key": "minute", "increment_value": 1, "ttl": 60}, + {"key": "day", "increment_value": 1, "ttl": 86400}, + ] + ) + now["t"] += 601 + + assert cache.get_cache("minute") is None + assert cache.get_cache("day") == 1 diff --git a/tests/unit/test_router/test_enforce_model_rate_limits.py b/tests/unit/test_router/test_enforce_model_rate_limits.py index 7577064b7f9..2a260fdf0b8 100644 --- a/tests/unit/test_router/test_enforce_model_rate_limits.py +++ b/tests/unit/test_router/test_enforce_model_rate_limits.py @@ -6,7 +6,7 @@ regardless of the routing strategy being used. """ import asyncio -from datetime import timedelta +from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock import pytest @@ -222,11 +222,9 @@ class TestModelRateLimitingCheck: check.log_success_event(kwargs, None, None, None) - # Verify increment_cache was called - mock_cache.increment_cache.assert_called_once() - _, kwarg_params = mock_cache.increment_cache.call_args - assert "test-id:gpt-4:tpm:" in kwarg_params["key"] - assert kwarg_params["value"] == 50 + keys = tuple(call.kwargs["key"] for call in mock_cache.increment_cache.call_args_list) + assert any(key.startswith("test-id:gpt-4:tpm:") for key in keys), keys + assert all(call.kwargs["value"] == 50 for call in mock_cache.increment_cache.call_args_list) class TestModelRateLimitingCheckAsync: @@ -253,7 +251,7 @@ class TestModelRateLimitingCheckAsync: """Test that RateLimitError is raised when RPM limit is exceeded (async).""" mock_cache = MagicMock() mock_cache.async_get_cache = AsyncMock(return_value=None) - mock_cache.async_increment_cache = AsyncMock(return_value=11) # Over limit + mock_cache.async_increment_cache_pipeline = AsyncMock(return_value=[11.0]) # Over limit check = ModelRateLimitingCheck(dual_cache=mock_cache) @@ -274,7 +272,7 @@ class TestModelRateLimitingCheckAsync: """Test that requests are allowed when under the limit (async).""" mock_cache = MagicMock() mock_cache.async_get_cache = AsyncMock(return_value=None) - mock_cache.async_increment_cache = AsyncMock(return_value=6) + mock_cache.async_increment_cache_pipeline = AsyncMock(return_value=[6.0]) check = ModelRateLimitingCheck(dual_cache=mock_cache) @@ -339,7 +337,7 @@ class TestModelRateLimitingCheckAsync: ): redis_cache = MagicMock() redis_cache.async_get_cache = AsyncMock(side_effect=RedisCircuitBreakerOpenError()) - redis_cache.async_increment = AsyncMock(return_value=2) + redis_cache.async_increment_pipeline = AsyncMock(return_value=[2.0]) check = ModelRateLimitingCheck(dual_cache=_dual_cache_with_local_tpm(5, redis_cache)) with pytest.raises(litellm.RateLimitError) as exc_info: @@ -375,11 +373,9 @@ class TestModelRateLimitingCheckAsync: await check.async_log_success_event(kwargs, None, None, None) - # Verify async_increment_cache was called - mock_cache.async_increment_cache.assert_called_once() - _, kwarg_params = mock_cache.async_increment_cache.call_args - assert "test-id:gpt-4:tpm:" in kwarg_params["key"] - assert kwarg_params["value"] == 50 + keys = tuple(call.kwargs["key"] for call in mock_cache.async_increment_cache.call_args_list) + assert any(key.startswith("test-id:gpt-4:tpm:") for key in keys), keys + assert all(call.kwargs["value"] == 50 for call in mock_cache.async_increment_cache.call_args_list) class TestRouterWithEnforceModelRateLimits: @@ -466,3 +462,265 @@ class TestModelRateLimitConcurrency: assert len(successes) == 2, f"Expected 2 successes, got {len(successes)}" assert len(failures) == 2, f"Expected 2 rate limit errors, got {len(failures)}" + + +DAY_ONE = datetime(2026, 10, 28, 23, 59, 30, tzinfo=timezone.utc) +RPD_DEPLOYMENT = { + "litellm_params": {"model": "gpt-4.1-mini", "rpd": 2}, + "model_info": {"id": "daily-test-id"}, + "model_name": "daily-model", +} +RPM_AND_RPD_DEPLOYMENT = {**RPD_DEPLOYMENT, "litellm_params": {"model": "gpt-4.1-mini", "rpm": 1, "rpd": 2}} + + +class Clock: + def __init__(self, now: datetime): + self.now = now + + def __call__(self) -> datetime: + return self.now + + +def _pre_call(check: ModelRateLimitingCheck, deployment: dict[str, object]) -> object: + return check.pre_call_check(deployment) + + +async def _async_pre_call(check: ModelRateLimitingCheck, deployment: dict[str, object]) -> object: + return await check.async_pre_call_check(deployment) + + +async def _async_log_success(check: ModelRateLimitingCheck, kwargs: dict[str, object]) -> None: + await check.async_log_success_event(kwargs, None, None, None) + + +class TestDailyLimits: + def test_pre_call_check_rejects_the_request_over_rpd(self): + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=Clock(DAY_ONE)) + + assert _pre_call(check, RPD_DEPLOYMENT) == RPD_DEPLOYMENT + assert _pre_call(check, RPD_DEPLOYMENT) == RPD_DEPLOYMENT + with pytest.raises(litellm.RateLimitError) as exc_info: + _pre_call(check, RPD_DEPLOYMENT) + + assert "RPD limit=2, current usage=3" in str(exc_info.value) + assert "rpd limit=2. current usage=3. id=daily-test-id, model_group=daily-model" in exc_info.value.response.text + + @pytest.mark.asyncio + async def test_async_pre_call_check_rejects_the_request_over_rpd_and_lets_the_router_retry(self): + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=Clock(DAY_ONE)) + + assert await _async_pre_call(check, RPD_DEPLOYMENT) == RPD_DEPLOYMENT + assert await _async_pre_call(check, RPD_DEPLOYMENT) == RPD_DEPLOYMENT + with pytest.raises(litellm.RateLimitError) as exc_info: + await _async_pre_call(check, RPD_DEPLOYMENT) + + assert "RPD limit=2" in str(exc_info.value) + assert exc_info.value.num_retries is None + + @pytest.mark.asyncio + async def test_async_pre_call_check_rejects_once_the_day_tokens_reach_tpd(self): + deployment = {**RPD_DEPLOYMENT, "litellm_params": {"model": "gpt-4.1-mini", "tpd": 100}} + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=Clock(DAY_ONE)) + kwargs = { + "standard_logging_object": { + "model_id": "daily-test-id", + "total_tokens": 60, + "hidden_params": {"litellm_model_name": "gpt-4.1-mini"}, + } + } + + await _async_log_success(check, kwargs) + assert await _async_pre_call(check, deployment) == deployment + await _async_log_success(check, kwargs) + with pytest.raises(litellm.RateLimitError) as exc_info: + await _async_pre_call(check, deployment) + + assert "TPD limit=100, current usage=120" in str(exc_info.value) + + def test_pre_call_check_minute_rejected_request_does_not_spend_rpd(self): + clock = Clock(DAY_ONE) + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock) + + assert _pre_call(check, RPM_AND_RPD_DEPLOYMENT) == RPM_AND_RPD_DEPLOYMENT + with pytest.raises(litellm.RateLimitError, match="RPM limit=1"): + _pre_call(check, RPM_AND_RPD_DEPLOYMENT) + clock.now = DAY_ONE - timedelta(minutes=1) + + assert _pre_call(check, RPM_AND_RPD_DEPLOYMENT) == RPM_AND_RPD_DEPLOYMENT + clock.now = DAY_ONE - timedelta(minutes=2) + with pytest.raises(litellm.RateLimitError, match="RPD limit=2, current usage=3"): + _pre_call(check, RPM_AND_RPD_DEPLOYMENT) + + @pytest.mark.asyncio + async def test_async_pre_call_check_minute_rejected_request_does_not_spend_rpd(self): + clock = Clock(DAY_ONE) + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock) + + assert await _async_pre_call(check, RPM_AND_RPD_DEPLOYMENT) == RPM_AND_RPD_DEPLOYMENT + with pytest.raises(litellm.RateLimitError, match="RPM limit=1"): + await _async_pre_call(check, RPM_AND_RPD_DEPLOYMENT) + clock.now = DAY_ONE - timedelta(minutes=1) + + assert await _async_pre_call(check, RPM_AND_RPD_DEPLOYMENT) == RPM_AND_RPD_DEPLOYMENT + clock.now = DAY_ONE - timedelta(minutes=2) + with pytest.raises(litellm.RateLimitError, match="RPD limit=2, current usage=3"): + await _async_pre_call(check, RPM_AND_RPD_DEPLOYMENT) + + def test_pre_call_check_rejected_requests_hand_back_rpm_and_rpd(self): + clock = Clock(DAY_ONE) + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock) + + assert _pre_call(check, RPM_AND_RPD_DEPLOYMENT) == RPM_AND_RPD_DEPLOYMENT + with pytest.raises(litellm.RateLimitError, match="RPM limit=1, current usage=2"): + _pre_call(check, RPM_AND_RPD_DEPLOYMENT) + with pytest.raises(litellm.RateLimitError, match="RPM limit=1, current usage=2"): + _pre_call(check, RPM_AND_RPD_DEPLOYMENT) + clock.now = DAY_ONE - timedelta(minutes=1) + assert _pre_call(check, RPM_AND_RPD_DEPLOYMENT) == RPM_AND_RPD_DEPLOYMENT + + clock.now = DAY_ONE - timedelta(minutes=2) + with pytest.raises(litellm.RateLimitError, match="RPD limit=2, current usage=3"): + _pre_call(check, RPM_AND_RPD_DEPLOYMENT) + with pytest.raises(litellm.RateLimitError, match="RPD limit=2, current usage=3"): + _pre_call(check, RPM_AND_RPD_DEPLOYMENT) + + @pytest.mark.asyncio + async def test_async_pre_call_check_rejected_requests_hand_back_rpm_and_rpd(self): + clock = Clock(DAY_ONE) + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock) + + assert await _async_pre_call(check, RPM_AND_RPD_DEPLOYMENT) == RPM_AND_RPD_DEPLOYMENT + with pytest.raises(litellm.RateLimitError, match="RPM limit=1, current usage=2"): + await _async_pre_call(check, RPM_AND_RPD_DEPLOYMENT) + with pytest.raises(litellm.RateLimitError, match="RPM limit=1, current usage=2"): + await _async_pre_call(check, RPM_AND_RPD_DEPLOYMENT) + clock.now = DAY_ONE - timedelta(minutes=1) + assert await _async_pre_call(check, RPM_AND_RPD_DEPLOYMENT) == RPM_AND_RPD_DEPLOYMENT + + clock.now = DAY_ONE - timedelta(minutes=2) + with pytest.raises(litellm.RateLimitError, match="RPD limit=2, current usage=3"): + await _async_pre_call(check, RPM_AND_RPD_DEPLOYMENT) + with pytest.raises(litellm.RateLimitError, match="RPD limit=2, current usage=3"): + await _async_pre_call(check, RPM_AND_RPD_DEPLOYMENT) + + @pytest.mark.asyncio + async def test_async_pre_call_check_refunds_rpm_and_rpd_in_one_cache_trip(self): + mock_cache = MagicMock() + mock_cache.async_get_cache = AsyncMock(return_value=None) + mock_cache.async_increment_cache_pipeline = AsyncMock(side_effect=[[1.0, 3.0], [0.0, 2.0]]) + check = ModelRateLimitingCheck(dual_cache=mock_cache, clock=Clock(DAY_ONE)) + + with pytest.raises(litellm.RateLimitError, match="RPD limit=2, current usage=3"): + await _async_pre_call(check, RPM_AND_RPD_DEPLOYMENT) + + assert mock_cache.async_increment_cache_pipeline.await_count == 2 + refund = mock_cache.async_increment_cache_pipeline.call_args_list[1].kwargs["increment_list"] + assert [(op["key"], op["increment_value"], op["ttl"]) for op in refund] == [ + ("daily-test-id:gpt-4.1-mini:rpm:23-59", -1, 60), + ("daily-test-id:gpt-4.1-mini:rpd:2026-10-28", -1, 86400), + ] + + @pytest.mark.asyncio + async def test_async_pre_call_check_increments_rpm_and_rpd_in_one_cache_trip(self): + mock_cache = MagicMock() + mock_cache.async_get_cache = AsyncMock(return_value=None) + mock_cache.async_increment_cache_pipeline = AsyncMock(return_value=[1.0, 1.0]) + check = ModelRateLimitingCheck(dual_cache=mock_cache, clock=Clock(DAY_ONE)) + + assert await _async_pre_call(check, RPM_AND_RPD_DEPLOYMENT) == RPM_AND_RPD_DEPLOYMENT + + assert mock_cache.async_increment_cache_pipeline.await_count == 1 + operations = mock_cache.async_increment_cache_pipeline.call_args.kwargs["increment_list"] + assert [(op["key"], op["ttl"]) for op in operations] == [ + ("daily-test-id:gpt-4.1-mini:rpm:23-59", 60), + ("daily-test-id:gpt-4.1-mini:rpd:2026-10-28", 86400), + ] + assert not mock_cache.async_increment_cache.called + + def test_pre_call_check_reads_the_shared_tpd_count_from_redis(self): + redis_cache = MagicMock() + redis_cache.get_cache.return_value = 100 + deployment = {**RPD_DEPLOYMENT, "litellm_params": {"model": "gpt-4.1-mini", "tpd": 100}} + check = ModelRateLimitingCheck(dual_cache=DualCache(redis_cache=redis_cache), clock=Clock(DAY_ONE)) + + with pytest.raises(litellm.RateLimitError) as exc_info: + _pre_call(check, deployment) + + assert "TPD limit=100, current usage=100" in str(exc_info.value) + assert redis_cache.get_cache.call_args.kwargs["key"] == "daily-test-id:gpt-4.1-mini:tpd:2026-10-28" + + def test_rpd_count_starts_fresh_after_utc_midnight(self): + clock = Clock(DAY_ONE) + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=clock) + _pre_call(check, RPD_DEPLOYMENT) + _pre_call(check, RPD_DEPLOYMENT) + with pytest.raises(litellm.RateLimitError): + _pre_call(check, RPD_DEPLOYMENT) + + clock.now = DAY_ONE + timedelta(seconds=30) + + assert _pre_call(check, RPD_DEPLOYMENT) == RPD_DEPLOYMENT + assert _pre_call(check, RPD_DEPLOYMENT) == RPD_DEPLOYMENT + with pytest.raises(litellm.RateLimitError) as exc_info: + _pre_call(check, RPD_DEPLOYMENT) + assert "current usage=3" in str(exc_info.value) + + @pytest.mark.parametrize( + ("now", "seconds_to_midnight"), + [ + (DAY_ONE, 30), + (datetime(2026, 10, 28, 0, 0, 0, tzinfo=timezone.utc), 86400), + (datetime(2026, 10, 28, 12, 30, 15, 500000, tzinfo=timezone.utc), 41384), + ], + ) + def test_daily_429_retry_after_is_seconds_until_utc_midnight(self, now: datetime, seconds_to_midnight: int): + deployment = {**RPD_DEPLOYMENT, "litellm_params": {"model": "gpt-4.1-mini", "rpd": 0}} + check = ModelRateLimitingCheck(dual_cache=DualCache(), clock=Clock(now)) + + with pytest.raises(litellm.RateLimitError) as exc_info: + _pre_call(check, deployment) + + assert exc_info.value.response.headers["retry-after"] == str(seconds_to_midnight) + + @pytest.mark.asyncio + async def test_router_serves_the_request_from_the_other_deployment_once_rpd_is_spent(self): + router = Router( + model_list=[ + { + "model_name": "daily-group", + "litellm_params": { + "model": "openai/gpt-4.1-mini", + "api_key": "k", + "mock_response": "a", + "rpd": 1, + "order": 1, + }, + "model_info": {"id": "spent-first"}, + }, + { + "model_name": "daily-group", + "litellm_params": { + "model": "openai/gpt-4.1-mini", + "api_key": "k", + "mock_response": "b", + "order": 2, + }, + "model_info": {"id": "always-free"}, + }, + ], + num_retries=1, + ) + check = ModelRateLimitingCheck(dual_cache=router.cache, clock=Clock(DAY_ONE)) + router.optional_callbacks = [check] + litellm.logging_callback_manager.add_litellm_callback(check) + try: + served = [ + (await router.acompletion(model="daily-group", messages=[{"role": "user", "content": "hi"}])) + .choices[0] + .message.content + for _ in range(4) + ] + finally: + router.discard() + + assert served == ["a", "b", "b", "b"] diff --git a/tests/unit/test_router/test_io_token_rate_limits.py b/tests/unit/test_router/test_io_token_rate_limits.py index a5a68271111..0ae22534278 100644 --- a/tests/unit/test_router/test_io_token_rate_limits.py +++ b/tests/unit/test_router/test_io_token_rate_limits.py @@ -859,7 +859,7 @@ class TestModelRateLimitingCheckIOTokens: with pytest.raises(litellm.RateLimitError): await check.async_pre_call_check(deployment) - assert await dual_cache.async_get_cache(key=rpm_key) == 6 + assert await dual_cache.async_get_cache(key=rpm_key) == 5, "the rejected request hands its rpm count back" assert (await dual_cache.async_get_cache(key=itpm_key) or 0) == 0 assert ITPM_RESERVED_KEY not in request_kwargs["metadata"] assert any("both limit types are enforced" in record.message for record in caplog.records) diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index 740342f54d9..cc1ea645012 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -154,6 +154,8 @@ OPTION_NAMES: Final = ( "model_info", "rpm", "tpm", + "rpd", + "tpd", "itpm", "otpm", "default_api_key_rpm_limit", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a09672d7d8f..a49f1099121 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35647,6 +35647,8 @@ export interface components { regional_processing_uplift_multiplier_us?: number | null; /** Responses Api Version */ responses_api_version?: string | null; + /** Rpd */ + rpd?: number | null; /** Rpm */ rpm?: number | null; /** S3 Access Key Id */ @@ -35699,6 +35701,8 @@ export interface components { token_exchange_profile?: string | null; /** Token Exchange Scope */ token_exchange_scope?: string | null; + /** Tpd */ + tpd?: number | null; /** Tpm */ tpm?: number | null; /** Use Chat Completions Api */ @@ -51039,6 +51043,8 @@ export interface components { regional_processing_uplift_multiplier_us?: number | null; /** Responses Api Version */ responses_api_version?: string | null; + /** Rpd */ + rpd?: number | null; /** Rpm */ rpm?: number | null; /** S3 Access Key Id */ @@ -51091,6 +51097,8 @@ export interface components { token_exchange_profile?: string | null; /** Token Exchange Scope */ token_exchange_scope?: string | null; + /** Tpd */ + tpd?: number | null; /** Tpm */ tpm?: number | null; /** Use Chat Completions Api */ From 8e8f6dc66ed693f21f60eddce1939fd179621832 Mon Sep 17 00:00:00 2001 From: tessellark <349031000+tessellark@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:53:00 -0700 Subject: [PATCH 16/27] feat(tencent): add Seedream image generation and editing through TokenHub (#48735) * feat(tencent): add Seedream image generation and editing through TokenHub * feat(tencent): price Seedream image generation and edits from TokenHub usage * test(tencent): cover Seedream image generation, edits, and pricing through TokenHub * fix(tencent): keep non-Seedream image models on the OpenAI-compatible route Only seedream- models use TokenHub's wand route, so Hy-Image and other tencent image models keep working as on main. The response now carries TokenHub's image size and format, Pro's flat fallback bills its default 2K size, and the new tests are typed. * fix(tencent): bill other Tencent image models as before and forward user and TokenHub params on edits * fix(tencent): keep dropped edit params out and parse form-string watermark and seed * fix(tencent): send extra_body edit fields to TokenHub as given * fix(tencent): keep the routed model on Seedream edits and bill a deployment's per-token price * fix(tencent): accept string drop params on edits, rebuild nested form fields, and honor a per-image deployment price An image edit with a non-list additional_drop_params answered 500 for every provider after this branch validated it as a list, so a non-list value now drops nothing again, as chat treats it. Seedream edits sent through the OpenAI SDK arrive as multipart form fields, so TokenHub-only booleans and integers are parsed and bracketed keys like optimize_prompt_options[mode] are rebuilt into objects. A deployment that sets only output_cost_per_image is billed that price, and the TokenHub URL uses the shared api_base helpers. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/images/main.py | 57 ++++- litellm/images/utils.py | 40 ++- .../litellm_core_utils/llm_cost_calc/utils.py | 83 +++---- .../flux_lora_depth_transformation.py | 5 +- .../llms/fal_ai/image_edit/transformation.py | 44 +--- litellm/llms/tencent/cost_calculator.py | 47 +++- litellm/llms/tencent/image_edit/__init__.py | 0 .../llms/tencent/image_edit/transformation.py | 147 +++++++++++ .../llms/tencent/image_generation/__init__.py | 0 .../image_generation/transformation.py | 200 +++++++++++++++ ...odel_prices_and_context_window_backup.json | 33 +++ .../provider_endpoints_support_backup.json | 4 +- litellm/utils.py | 16 ++ model_prices_and_context_window.json | 33 +++ provider_endpoints_support.json | 4 +- tests/unit/images/test_main.py | 18 ++ .../unit/llms/tencent/image_edit/__init__.py | 0 .../test_tencent_image_edit_transformation.py | 232 ++++++++++++++++++ .../llms/tencent/image_generation/__init__.py | 0 ...tencent_image_generation_transformation.py | 215 ++++++++++++++++ .../tencent/test_tencent_cost_calculator.py | 128 ++++++++++ 21 files changed, 1204 insertions(+), 102 deletions(-) create mode 100644 litellm/llms/tencent/image_edit/__init__.py create mode 100644 litellm/llms/tencent/image_edit/transformation.py create mode 100644 litellm/llms/tencent/image_generation/__init__.py create mode 100644 litellm/llms/tencent/image_generation/transformation.py create mode 100644 tests/unit/llms/tencent/image_edit/__init__.py create mode 100644 tests/unit/llms/tencent/image_edit/test_tencent_image_edit_transformation.py create mode 100644 tests/unit/llms/tencent/image_generation/__init__.py create mode 100644 tests/unit/llms/tencent/image_generation/test_tencent_image_generation_transformation.py create mode 100644 tests/unit/llms/tencent/test_tencent_cost_calculator.py diff --git a/litellm/images/main.py b/litellm/images/main.py index 84156919774..d0618e5e804 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -1,9 +1,9 @@ import asyncio import contextvars import importlib -from collections.abc import Coroutine, Mapping +from collections.abc import Collection, Coroutine, Mapping from functools import partial -from typing import TYPE_CHECKING, Final, Literal, cast, overload +from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload from pydantic import TypeAdapter, ValidationError @@ -61,6 +61,7 @@ from litellm.types.utils import ( ProviderRoutingHints, ) from litellm.utils import ( + EXTRA_BODY_ROUTING_KEYS, ImageResponse, ProviderConfigManager, get_llm_provider, @@ -466,7 +467,7 @@ def image_generation( litellm.LlmProviders.QWENCLOUD, litellm.LlmProviders.QWEN_AI_PLATFORM, litellm.LlmProviders.EDENAI, - ): + ) or (custom_llm_provider == litellm.LlmProviders.TENCENT and image_generation_config is not None): if image_generation_config is None: raise ValueError(f"image generation config is not supported for {custom_llm_provider}") @@ -807,6 +808,32 @@ def image_variation( return response +_IMAGE_EDIT_OPENAI_PARAMS: Final = frozenset(get_type_hints(ImageEditOptionalRequestParams)) +_DROP_PARAM_KWARGS: Final = frozenset({"drop_params", "additional_drop_params"}) +_ADDITIONAL_DROP_PARAMS: Final = TypeAdapter[tuple[str, ...]](tuple[str, ...]) + + +def _listed_drop_params(additional_drop_params: object) -> tuple[str, ...]: + try: + return _ADDITIONAL_DROP_PARAMS.validate_python(additional_drop_params) + except ValidationError: + return () + + +def _tokenhub_passthrough_edit_params( + non_default_params: Mapping[str, object], + extra_body: Mapping[str, object] | None, + mapped_params: Collection[str], + additional_drop_params: Collection[str] | None, +) -> Mapping[str, object]: + dropped: Final = frozenset(additional_drop_params or ()) + unmapped_excluded: Final = _DROP_PARAM_KWARGS | _IMAGE_EDIT_OPENAI_PARAMS | frozenset(mapped_params) | dropped + return { + **{key: value for key, value in non_default_params.items() if key not in unmapped_excluded}, + **{key: value for key, value in (extra_body or {}).items() if key not in dropped | EXTRA_BODY_ROUTING_KEYS}, + } + + @client def image_edit( image: FileTypes | list[FileTypes] | None = None, @@ -935,22 +962,24 @@ def image_edit( raise ValueError(f"image edit is not supported for {custom_llm_provider}") local_vars.update(kwargs) + provider_supported_params: Final[frozenset[str]] = frozenset( + image_edit_provider_config.get_supported_openai_params(model) + ) # Get ImageEditOptionalRequestParams with only valid parameters image_edit_optional_params: Final[ImageEditOptionalRequestParams] = ( _get_ImageEditRequestUtils().get_requested_image_edit_optional_param( local_vars, - provider_supported_params=frozenset( - image_edit_provider_config.get_supported_openai_params(model) - ).intersection(non_default_params), + provider_supported_params=provider_supported_params.intersection(non_default_params), ) ) # Get optional parameters for the responses API + additional_drop_params: Final = _listed_drop_params(kwargs.get("additional_drop_params")) image_edit_request_params: Final[dict] = _get_ImageEditRequestUtils().get_optional_params_image_edit( model=model, image_edit_provider_config=image_edit_provider_config, image_edit_optional_params=image_edit_optional_params, drop_params=kwargs.get("drop_params"), - additional_drop_params=kwargs.get("additional_drop_params"), + additional_drop_params=additional_drop_params, ) if image_edit_provider_config.use_multipart_form_data() and ( @@ -1039,7 +1068,19 @@ def image_edit( image=images, prompt=prompt, image_edit_provider_config=image_edit_provider_config, - image_edit_optional_request_params=image_edit_request_params, + image_edit_optional_request_params={ + **image_edit_request_params, + **( + _tokenhub_passthrough_edit_params( + non_default_params, + extra_body if isinstance(extra_body, dict) else None, + provider_supported_params, + additional_drop_params, + ) + if custom_llm_provider == litellm.LlmProviders.TENCENT.value + else {} + ), + }, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params, logging_obj=litellm_logging_obj, diff --git a/litellm/images/utils.py b/litellm/images/utils.py index 24454954714..4e0cde24a68 100644 --- a/litellm/images/utils.py +++ b/litellm/images/utils.py @@ -1,6 +1,9 @@ +import base64 +import os from collections.abc import Collection, Mapping from io import BufferedReader, BytesIO -from typing import Any, Final, cast, get_type_hints +from pathlib import Path +from typing import Any, Final, Protocol, cast, get_type_hints, runtime_checkable import litellm from litellm.litellm_core_utils.token_counter import get_image_type @@ -16,7 +19,7 @@ class ImageEditRequestUtils: image_edit_provider_config: BaseImageEditConfig, image_edit_optional_params: ImageEditOptionalRequestParams, drop_params: bool | None = None, - additional_drop_params: list[str] | None = None, + additional_drop_params: Collection[str] | None = None, ) -> dict: """ Get optional parameters for the image edit API. @@ -145,3 +148,36 @@ class ImageEditRequestUtils: except Exception: # If anything goes wrong, default to PNG return FILE_MIME_TYPES[FileType.PNG] + + +@runtime_checkable +class _SeekableBinaryReader(Protocol): + def tell(self) -> int: ... + + def seek(self, offset: int) -> int: ... + + def read(self) -> bytes: ... + + +def _read_image_bytes(image: object) -> bytes: + if isinstance(image, bytes): + return image + if isinstance(image, tuple): + return _read_image_bytes(image[1]) + if isinstance(image, os.PathLike): + return Path(image).read_bytes() + if isinstance(image, _SeekableBinaryReader): + position: Final = image.tell() + image.seek(0) + data: Final = image.read() + image.seek(position) + return data + raise ValueError(f"Unsupported image type for image edit: {type(image).__name__}") + + +def image_to_data_url(image: object) -> str: + if isinstance(image, str): + return image + image_bytes: Final = _read_image_bytes(image) + mime_type: Final = ImageEditRequestUtils.get_image_content_type(image_bytes) + return f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('utf-8')}" diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 3b49f239afe..4b0b875af6d 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -6,7 +6,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace from datetime import datetime, timezone, tzinfo from types import MappingProxyType -from typing import Final, Literal, TypedDict, cast +from typing import Final, Literal, Protocol, TypedDict, cast from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from typing_extensions import ReadOnly @@ -2085,6 +2085,41 @@ def calculate_image_response_web_search_cost( ) +class _ModelPricedImageCostCalculator(Protocol): + def __call__(self, model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float: ... + + +def _model_priced_image_cost_calculator( + custom_llm_provider: str | None, model: str +) -> _ModelPricedImageCostCalculator | None: + match custom_llm_provider: + case litellm.LlmProviders.RECRAFT.value: + from litellm.llms.recraft.cost_calculator import cost_calculator as recraft_image_cost_calculator + + return recraft_image_cost_calculator + case litellm.LlmProviders.AIML.value: + from litellm.llms.aiml.image_generation.cost_calculator import cost_calculator as aiml_image_cost_calculator + + return aiml_image_cost_calculator + case litellm.LlmProviders.COMETAPI.value: + from litellm.llms.cometapi.image_generation.cost_calculator import ( + cost_calculator as cometapi_image_cost_calculator, + ) + + return cometapi_image_cost_calculator + case litellm.LlmProviders.RUNWAYML.value: + from litellm.llms.runwayml.cost_calculator import cost_calculator as runwayml_image_cost_calculator + + return runwayml_image_cost_calculator + case litellm.LlmProviders.TENCENT.value: + from litellm.llms.tencent.cost_calculator import image_cost_calculator as tencent_image_cost_calculator + from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model + + return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None + case _: + return None + + class CostCalculatorUtils: @staticmethod def call_type_has_image_response(call_type: str) -> bool: @@ -2131,9 +2166,6 @@ class CostCalculatorUtils: from litellm.llms.gemini.image_generation.cost_calculator import ( cost_calculator as gemini_image_cost_calculator, ) - from litellm.llms.recraft.cost_calculator import ( - cost_calculator as recraft_image_cost_calculator, - ) from litellm.llms.vertex_ai.image_generation.cost_calculator import ( cost_calculator as vertex_ai_image_cost_calculator, ) @@ -2146,7 +2178,10 @@ class CostCalculatorUtils: ) resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0) pricing: Final = deployment_pricing(model_info) + model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model) + if model_priced_calculator is not None: + return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing) if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value: if isinstance(completion_response, ImageResponse): return vertex_ai_image_cost_calculator( @@ -2164,36 +2199,6 @@ class CostCalculatorUtils: optional_params=optional_params, ) raise TypeError("completion_response must be of type ImageResponse for bedrock image cost calculation") - elif custom_llm_provider == litellm.LlmProviders.RECRAFT.value: - from litellm.llms.recraft.cost_calculator import ( - cost_calculator as recraft_image_cost_calculator, - ) - - return recraft_image_cost_calculator( - model=model, - image_response=completion_response, - model_info=pricing, - ) - elif custom_llm_provider == litellm.LlmProviders.AIML.value: - from litellm.llms.aiml.image_generation.cost_calculator import ( - cost_calculator as aiml_image_cost_calculator, - ) - - return aiml_image_cost_calculator( - model=model, - image_response=completion_response, - model_info=pricing, - ) - elif custom_llm_provider == litellm.LlmProviders.COMETAPI.value: - from litellm.llms.cometapi.image_generation.cost_calculator import ( - cost_calculator as cometapi_image_cost_calculator, - ) - - return cometapi_image_cost_calculator( - model=model, - image_response=completion_response, - model_info=pricing, - ) elif custom_llm_provider == litellm.LlmProviders.GEMINI.value: if call_type in ( CallTypes.image_edit.value, @@ -2237,16 +2242,6 @@ class CostCalculatorUtils: optional_params=optional_params, model_info=pricing, ) - elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value: - from litellm.llms.runwayml.cost_calculator import ( - cost_calculator as runwayml_image_cost_calculator, - ) - - return runwayml_image_cost_calculator( - model=model, - image_response=completion_response, - model_info=pricing, - ) elif ( custom_llm_provider == litellm.LlmProviders.OPENAI.value or custom_llm_provider == litellm.LlmProviders.AZURE.value diff --git a/litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py b/litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py index 344d9d3c75b..6d0891295bd 100644 --- a/litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py +++ b/litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py @@ -4,13 +4,14 @@ from typing import Final from httpx._types import RequestFiles +from litellm.images.utils import image_to_data_url from litellm.litellm_core_utils.url_utils import join_api_base from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import FileTypes -from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig, to_data_url +from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig FLUX_LORA_DEPTH_ENDPOINT: Final[str] = "fal-ai/flux-lora-depth" SUPPORTED_OPENAI_PARAMS: Final[tuple[str, ...]] = ("n", "size") @@ -68,7 +69,7 @@ class FalAIFluxLoraDepthEditConfig(FalAIImageEditConfig): ) request_body: Final[dict[str, object]] = { # mutable-ok: base class contract returns a dict "prompt": prompt, - "image_url": to_data_url(next(iter(images))), + "image_url": image_to_data_url(next(iter(images))), **provider_params, } return request_body, () diff --git a/litellm/llms/fal_ai/image_edit/transformation.py b/litellm/llms/fal_ai/image_edit/transformation.py index ea4e8cac5bf..5cbf635fd47 100644 --- a/litellm/llms/fal_ai/image_edit/transformation.py +++ b/litellm/llms/fal_ai/image_edit/transformation.py @@ -1,15 +1,12 @@ -import base64 -import os from collections.abc import Mapping -from pathlib import Path from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Final import httpx from httpx._types import RequestFiles from pydantic import ConfigDict, TypeAdapter -from litellm.images.utils import ImageEditRequestUtils +from litellm.images.utils import image_to_data_url from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.llms.fal_ai.image_generation.gpt_image_2_transformation import ( map_gpt_image_quality, @@ -39,39 +36,6 @@ PARAM_TRANSLATION: Final[Mapping[str, str]] = MappingProxyType( ) -@runtime_checkable -class _SeekableBinaryReader(Protocol): - def tell(self) -> int: ... - - def seek(self, offset: int) -> int: ... - - def read(self) -> bytes: ... - - -def _read_image_bytes(image: object) -> bytes: - if isinstance(image, bytes): - return image - if isinstance(image, tuple): - return _read_image_bytes(image[1]) - if isinstance(image, os.PathLike): - return Path(image).read_bytes() - if isinstance(image, _SeekableBinaryReader): - position: Final = image.tell() - image.seek(0) - data: Final = image.read() - image.seek(position) - return data - raise ValueError(f"Unsupported image type for Fal AI image edit: {type(image).__name__}") - - -def to_data_url(image: object) -> str: - if isinstance(image, str): - return image - image_bytes: Final = _read_image_bytes(image) - mime_type: Final = ImageEditRequestUtils.get_image_content_type(image_bytes) - return f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('utf-8')}" - - def _first(value: object) -> object: return value[0] if isinstance(value, list) and value else value @@ -146,14 +110,14 @@ class FalAIImageEditConfig(BaseImageEditConfig): raise ValueError("Fal AI image edit requires at least one input image") mask: Final = _first(image_edit_optional_request_params.get("mask")) mask_field: Final[Mapping[str, str]] = ( - MappingProxyType({"mask_url": to_data_url(mask)}) if mask is not None else MappingProxyType({}) + MappingProxyType({"mask_url": image_to_data_url(mask)}) if mask is not None else MappingProxyType({}) ) provider_params: Final[Mapping[str, object]] = MappingProxyType( {key: value for key, value in image_edit_optional_request_params.items() if key != "mask"} ) request_body: Final[dict[str, object]] = { # mutable-ok: base class contract returns a dict "prompt": prompt, - "image_urls": tuple(to_data_url(img) for img in images), + "image_urls": tuple(image_to_data_url(img) for img in images), **mask_field, **provider_params, } diff --git a/litellm/llms/tencent/cost_calculator.py b/litellm/llms/tencent/cost_calculator.py index d9aebdc3284..5d2abe3acf4 100644 --- a/litellm/llms/tencent/cost_calculator.py +++ b/litellm/llms/tencent/cost_calculator.py @@ -1,6 +1,49 @@ -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token -from litellm.types.utils import Usage +from typing import Final + +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + calculate_image_response_cost_from_usage, + flat_image_cost, + generic_cost_per_token, + resolve_image_model_info, +) +from litellm.types.utils import ImageResponse, ModelInfo, Usage def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="tencent") + + +def _deployment_image_token_pricing(model_info: ModelInfo | None) -> ModelInfo | None: + if model_info is None or model_info.get("output_cost_per_image_token") is not None: + return model_info + output_cost_per_token: Final = model_info.get("output_cost_per_token") + if output_cost_per_token is None: + return model_info + priced: Final[ModelInfo] = {**model_info, "output_cost_per_image_token": output_cost_per_token} + return priced + + +def _deployment_prices_per_image_only(model_info: ModelInfo | None) -> bool: + return ( + model_info is not None + and model_info.get("output_cost_per_image") is not None + and model_info.get("output_cost_per_image_token") is None + and model_info.get("output_cost_per_token") is None + ) + + +def image_cost_calculator(model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float: + resolved_model_info: Final = resolve_image_model_info( + model=model, custom_llm_provider="tencent", model_info=_deployment_image_token_pricing(model_info) + ) + if _deployment_prices_per_image_only(model_info): + return flat_image_cost(resolved_model_info, image_response) + token_based_cost: Final = calculate_image_response_cost_from_usage( + model=model, + image_response=image_response, + custom_llm_provider="tencent", + model_info=resolved_model_info, + ) + if token_based_cost is not None: + return token_based_cost + return flat_image_cost(resolved_model_info, image_response) diff --git a/litellm/llms/tencent/image_edit/__init__.py b/litellm/llms/tencent/image_edit/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/tencent/image_edit/transformation.py b/litellm/llms/tencent/image_edit/transformation.py new file mode 100644 index 00000000000..3033d161ce7 --- /dev/null +++ b/litellm/llms/tencent/image_edit/transformation.py @@ -0,0 +1,147 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeAlias + +import httpx +from httpx._types import RequestFiles +from pydantic import TypeAdapter +from typing_extensions import override + +from litellm.images.utils import image_to_data_url +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.tencent.image_generation.transformation import ( + TOKENHUB_IMAGE_PARAMS, + tokenhub_image_headers, + tokenhub_image_params, + tokenhub_image_response, + tokenhub_image_url, +) +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import ImageResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +_IMAGES: Final = TypeAdapter(tuple[object, ...]) +_FORM_BOOLEANS: Final = MappingProxyType({"true": True, "false": False}) +_OPENAI_PARAMS: Final = frozenset(TOKENHUB_IMAGE_PARAMS) + +_FormEntries: TypeAlias = tuple[tuple[tuple[str, ...], object], ...] + + +def _form_scalar(value: object) -> object: + match value: + case str() if value.lower() in _FORM_BOOLEANS: + return _FORM_BOOLEANS[value.lower()] + case str() if value.removeprefix("-").isdigit(): + return int(value) + case _: + return value + + +def _form_path(key: str) -> tuple[str, ...]: + root, bracket, rest = key.partition("[") + if not bracket or not rest.endswith("]"): + return (key,) + return (root, *rest.removesuffix("]").split("][")) + + +def _nested_form_value(entries: _FormEntries) -> object: + match entries: + case (((), value),): + return _form_scalar(value) + case _: + return _nested_form_fields(tuple(entry for entry in entries if entry[0])) + + +def _nested_form_fields(entries: _FormEntries) -> Mapping[str, object]: + roots: Final = tuple(dict.fromkeys(path[0] for path, _ in entries)) + return { + root: _nested_form_value(tuple((path[1:], value) for path, value in entries if path[0] == root)) + for root in roots + } + + +def _tokenhub_form_params(params: Mapping[str, object]) -> Mapping[str, object]: + tokenhub_entries: Final = tuple( + (_form_path(key), value) for key, value in params.items() if key not in _OPENAI_PARAMS + ) + return { + **{key: value for key, value in params.items() if key in _OPENAI_PARAMS}, + **_nested_form_fields(tokenhub_entries), + } + + +class TencentImageEditConfig(BaseImageEditConfig): + @override + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract + return list(TOKENHUB_IMAGE_PARAMS) + + @override + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: mirrors override contract + return dict(tokenhub_image_params(image_edit_optional_params, model, drop_params)) + + @override + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + api_key: str | None = None, + litellm_params: Mapping[str, object] | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: mirrors override contract + return dict(tokenhub_image_headers(headers, api_key, model)) + + @override + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: Mapping[str, object], + ) -> str: + return tokenhub_image_url(api_base) + + @override + def use_multipart_form_data(self) -> bool: + return False + + @override + def transform_image_edit_request( + self, + model: str, + prompt: str | None, + image: object, + image_edit_optional_request_params: Mapping[str, object], + litellm_params: GenericLiteLLMParams, + headers: Mapping[str, str], + ) -> tuple[dict[str, object], RequestFiles]: # mutable-ok: mirrors override contract + images: Final = _IMAGES.validate_python(image) if isinstance(image, list) else (image,) + return { + "model": model, + "prompt": prompt, + "images": tuple(image_to_data_url(img) for img in images if img is not None), + **_tokenhub_form_params(image_edit_optional_request_params), + }, () + + @override + def transform_image_edit_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> ImageResponse: + image_response: Final = tokenhub_image_response(raw_response) + if image_response is None: + raise BaseLLMException( + status_code=raw_response.status_code, + message=raw_response.text, + headers=raw_response.headers, + ) + return image_response diff --git a/litellm/llms/tencent/image_generation/__init__.py b/litellm/llms/tencent/image_generation/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/tencent/image_generation/transformation.py b/litellm/llms/tencent/image_generation/transformation.py new file mode 100644 index 00000000000..fb4b102e8be --- /dev/null +++ b/litellm/llms/tencent/image_generation/transformation.py @@ -0,0 +1,200 @@ +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Final + +import httpx +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from typing_extensions import override + +from litellm.exceptions import AuthenticationError, UnsupportedParamsError +from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIImageGenerationOptionalParams, +) +from litellm.types.utils import ( + ImageObject, + ImageResponse, + ImageUsage, + ImageUsageInputTokensDetails, + LlmProviders, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +TOKENHUB_API_BASE: Final = "https://tokenhub-intl.tencentcloudmaas.com/v1" +TOKENHUB_IMAGE_PATH: Final = "/wand/si-image/generation" +TOKENHUB_IMAGE_MODEL_PREFIX: Final = "seedream-" +TOKENHUB_IMAGE_PARAMS: Final[tuple[OpenAIImageGenerationOptionalParams, ...]] = ( + "n", + "size", + "response_format", + "output_format", + "background", + "user", +) + +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) + + +class _TokenHubImage(BaseModel): + model_config = ConfigDict(frozen=True) + + url: str | None = None + b64_json: str | None = None + size: str | None = None + output_format: str | None = None + + +class _TokenHubUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + total_tokens: int + + +class _TokenHubImageResponse(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + created: int + data: tuple[_TokenHubImage, ...] + tokenhub_usage: _TokenHubUsage | None = None + + +def is_tokenhub_image_model(model: str) -> bool: + return model.removeprefix(f"{LlmProviders.TENCENT.value}/").startswith(TOKENHUB_IMAGE_MODEL_PREFIX) + + +def tokenhub_image_url(api_base: str | None) -> str: + base: Final = api_base or get_secret_str("TENCENT_API_BASE") or TOKENHUB_API_BASE + return join_api_base(ensure_api_route(base, "v1"), TOKENHUB_IMAGE_PATH) + + +def tokenhub_image_headers(headers: Mapping[str, str], api_key: str | None, model: str) -> Mapping[str, str]: + resolved_api_key: Final = api_key or get_secret_str("TENCENT_API_KEY") + if not resolved_api_key: + raise AuthenticationError( + message="TENCENT_API_KEY is not set. Set it in the environment or pass api_key.", + llm_provider="tencent", + model=model, + ) + return {**headers, "Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"} + + +def tokenhub_image_params(params: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, object]: + n: Final = params.get("n") + if n is not None and n != 1 and not drop_params: + raise UnsupportedParamsError( + status_code=400, + message=f"Tencent TokenHub returns one image per request, so n={n} is not supported for {model}. " + + "To send n=1 instead, set `litellm.drop_params = True`.", + ) + return { + key: value + for key, value in params.items() + if key != "n" and value is not None and not (key == "background" and value == "auto") + } + + +def tokenhub_image_response(raw_response: httpx.Response) -> ImageResponse | None: + try: + parsed: Final = _TokenHubImageResponse.model_validate_json(raw_response.content) + except ValidationError: + return None + total_tokens: Final = parsed.tokenhub_usage.total_tokens if parsed.tokenhub_usage is not None else 0 + first_image: Final = parsed.data[0] if parsed.data else _TokenHubImage() + return ImageResponse( + created=parsed.created, + data=[ImageObject(url=image.url, b64_json=image.b64_json) for image in parsed.data], + size=first_image.size, + output_format=first_image.output_format, + usage=ImageUsage( + input_tokens=0, + input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=0), + output_tokens=total_tokens, + total_tokens=total_tokens, + ), + ) + + +class TencentImageGenerationConfig(BaseImageGenerationConfig): + @override + def get_supported_openai_params( + self, model: str + ) -> list[OpenAIImageGenerationOptionalParams]: # mutable-ok: mirrors override contract + return list(TOKENHUB_IMAGE_PARAMS) + + @override + def map_openai_params( + self, + non_default_params: Mapping[str, object], + optional_params: Mapping[str, object], + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: mirrors override contract + return {**optional_params, **tokenhub_image_params(non_default_params, model, drop_params)} + + @override + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + stream: bool | None = None, + ) -> str: + return tokenhub_image_url(api_base) + + @override + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + messages: Sequence[AllMessageValues], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: mirrors override contract + return dict(tokenhub_image_headers(headers, api_key, model)) + + @override + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + headers: Mapping[str, str], + ) -> dict[str, object]: # mutable-ok: mirrors override contract + extra_body: Final = _JSON_OBJECT.validate_python(optional_params.get("extra_body") or {}) + params: Final = {key: value for key, value in optional_params.items() if key != "extra_body"} + return {"model": model, "prompt": prompt, **params, **extra_body} + + @override + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: "LiteLLMLoggingObj", + request_data: Mapping[str, object], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + encoding: object, + api_key: str | None = None, + json_mode: bool | None = None, + ) -> ImageResponse: + image_response: Final = tokenhub_image_response(raw_response) + if image_response is None: + raise BaseLLMException( + status_code=raw_response.status_code, + message=raw_response.text, + headers=raw_response.headers, + ) + return image_response diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1f2bd7d7837..dc043dc4d26 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -61986,6 +61986,39 @@ "supports_reasoning": true, "supports_vision": false }, + "tencent/seedream-image-v5.0-flash": { + "litellm_provider": "tencent", + "mode": "image_generation", + "output_cost_per_image": 0.018, + "output_cost_per_image_token": 1.6e-06, + "source": "https://cloud.tencent.com/document/product/1823/130055", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "tencent/seedream-image-v5.0-lite": { + "litellm_provider": "tencent", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_image_token": 1.6e-06, + "source": "https://www.tencentcloud.com/document/product/1300/78937", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "tencent/seedream-image-v5.0-pro": { + "litellm_provider": "tencent", + "mode": "image_generation", + "output_cost_per_image": 0.09, + "output_cost_per_image_token": 1.6e-06, + "source": "https://www.tencentcloud.com/document/product/1300/78937", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, "cognition/swe-1.6": { "input_cost_per_token": 5e-07, "output_cost_per_token": 2.5e-06, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index d73ac19bae1..0a6c2bcb2a9 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -3793,8 +3793,8 @@ "responses": true, "text_completion": true, "embeddings": false, - "image_generations": false, - "image_edits": false, + "image_generations": true, + "image_edits": true, "audio_transcriptions": false, "audio_speech": false, "rerank": false, diff --git a/litellm/utils.py b/litellm/utils.py index a231644addf..1ef39059e1b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10418,6 +10418,13 @@ class ProviderConfigManager: return get_modelscope_image_generation_config(model) elif LlmProviders.EDENAI == provider: return litellm.EdenAIImageGenerationConfig() + elif LlmProviders.TENCENT == provider: + from litellm.llms.tencent.image_generation.transformation import ( + TencentImageGenerationConfig, + is_tokenhub_image_model, + ) + + return TencentImageGenerationConfig() if is_tokenhub_image_model(model) else None return None @staticmethod @@ -10606,6 +10613,15 @@ class ProviderConfigManager: ) return get_openrouter_image_edit_config(model) + elif LlmProviders.TENCENT == provider: + from litellm.llms.tencent.image_edit.transformation import ( + TencentImageEditConfig, + ) + from litellm.llms.tencent.image_generation.transformation import ( + is_tokenhub_image_model, + ) + + return TencentImageEditConfig() if is_tokenhub_image_model(model) else None return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1f2bd7d7837..dc043dc4d26 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -61986,6 +61986,39 @@ "supports_reasoning": true, "supports_vision": false }, + "tencent/seedream-image-v5.0-flash": { + "litellm_provider": "tencent", + "mode": "image_generation", + "output_cost_per_image": 0.018, + "output_cost_per_image_token": 1.6e-06, + "source": "https://cloud.tencent.com/document/product/1823/130055", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "tencent/seedream-image-v5.0-lite": { + "litellm_provider": "tencent", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_image_token": 1.6e-06, + "source": "https://www.tencentcloud.com/document/product/1300/78937", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "tencent/seedream-image-v5.0-pro": { + "litellm_provider": "tencent", + "mode": "image_generation", + "output_cost_per_image": 0.09, + "output_cost_per_image_token": 1.6e-06, + "source": "https://www.tencentcloud.com/document/product/1300/78937", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, "cognition/swe-1.6": { "input_cost_per_token": 5e-07, "output_cost_per_token": 2.5e-06, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index d73ac19bae1..0a6c2bcb2a9 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -3793,8 +3793,8 @@ "responses": true, "text_completion": true, "embeddings": false, - "image_generations": false, - "image_edits": false, + "image_generations": true, + "image_edits": true, "audio_transcriptions": false, "audio_speech": false, "rerank": false, diff --git a/tests/unit/images/test_main.py b/tests/unit/images/test_main.py index 88d5ae1fc2e..29424e50079 100644 --- a/tests/unit/images/test_main.py +++ b/tests/unit/images/test_main.py @@ -357,6 +357,24 @@ def test_image_edit_prices_a_vertex_deployment_at_its_configured_location( assert cost_at("us-central1") == pytest.approx(0.044) +def test_image_edit_accepts_a_string_additional_drop_params(respx_mock: respx.MockRouter) -> None: + edits_route: Final = respx_mock.post("https://api.openai.com/v1/images/edits").mock( + return_value=httpx.Response(status_code=200, json={"created": 1792551838, "data": [{"b64_json": "aW1n"}]}) + ) + + response: Final = litellm.image_edit( + model="openai/gpt-image-2.5-sunburst", + image=b"\x89PNG\r\n\x1a\nfakepng", + prompt="make the circle blue", + api_key="sk-test", + quality="high", + additional_drop_params="quality", + ) + + assert edits_route.called + assert response.data[0].b64_json == "aW1n" + + class TestCustomLogger(CustomLogger): __test__ = False diff --git a/tests/unit/llms/tencent/image_edit/__init__.py b/tests/unit/llms/tencent/image_edit/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/tencent/image_edit/test_tencent_image_edit_transformation.py b/tests/unit/llms/tencent/image_edit/test_tencent_image_edit_transformation.py new file mode 100644 index 00000000000..506c43f6cba --- /dev/null +++ b/tests/unit/llms/tencent/image_edit/test_tencent_image_edit_transformation.py @@ -0,0 +1,232 @@ +import base64 +import io +import json +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler + +PNG_BYTES: Final = b"\x89PNG\r\n\x1a\n" + b"\x00" * 32 +TOKENHUB_RESPONSE: Final[dict[str, object]] = { + "created": 1792551838, + "data": [{"output_format": "jpeg", "size": "2048x2048", "url": "https://cos.example/edited.jpg"}], + "model": "doubao-seedream-5-0-lite", + "tokenhub_usage": {"total_tokens": 21875}, + "request_id": "4f795d48-b227-4c35-af2c-1a0ba288b6b6", +} + + +def _client(captured: dict[str, object]) -> HTTPHandler: + def respond(request: httpx.Request) -> httpx.Response: + captured["url"] = str(request.url) + captured["content_type"] = request.headers.get("content-type") + captured["authorization"] = request.headers.get("authorization") + captured["body"] = json.loads(request.content) + return httpx.Response(200, json=TOKENHUB_RESPONSE) + + return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + + +def test_edit_sends_uploaded_images_as_data_urls_in_a_json_body() -> None: + captured: Final[dict[str, object]] = {} + png_data_url: Final = f"data:image/png;base64,{base64.b64encode(PNG_BYTES).decode()}" + + response: Final = litellm.image_edit( + model="tencent/seedream-image-v5.0-lite", + image=[PNG_BYTES, io.BytesIO(PNG_BYTES), "https://ref.example/style.png"], + prompt="turn the square into a paper lantern", + size="2K", + api_key="sk-tokenhub", + client=_client(captured), + ) + + assert captured["url"] == "https://tokenhub-intl.tencentcloudmaas.com/v1/wand/si-image/generation" + assert captured["content_type"] == "application/json" + assert captured["authorization"] == "Bearer sk-tokenhub" + assert captured["body"] == { + "model": "seedream-image-v5.0-lite", + "prompt": "turn the square into a paper lantern", + "images": [png_data_url, png_data_url, "https://ref.example/style.png"], + "size": "2K", + } + assert response.data[0].url == "https://cos.example/edited.jpg" + assert response.usage.total_tokens == 21875 + + +def test_edit_rejects_a_mask_without_drop_params() -> None: + captured: Final[dict[str, object]] = {} + + with pytest.raises(litellm.UnsupportedParamsError): + litellm.image_edit( + model="tencent/seedream-image-v5.0-pro", + image=PNG_BYTES, + mask=PNG_BYTES, + prompt="turn the square into a paper lantern", + api_key="sk-tokenhub", + client=_client(captured), + ) + + assert captured == {} + + +def test_edit_drops_a_mask_under_drop_params() -> None: + captured: Final[dict[str, object]] = {} + + litellm.image_edit( + model="tencent/seedream-image-v5.0-pro", + image=PNG_BYTES, + mask=PNG_BYTES, + prompt="turn the square into a paper lantern", + api_key="sk-tokenhub", + client=_client(captured), + drop_params=True, + ) + + assert captured["body"] == { + "model": "seedream-image-v5.0-pro", + "prompt": "turn the square into a paper lantern", + "images": [f"data:image/png;base64,{base64.b64encode(PNG_BYTES).decode()}"], + } + + +def test_edit_is_not_offered_for_tencent_models_outside_seedream() -> None: + captured: Final[dict[str, object]] = {} + + with pytest.raises(Exception, match="image edit is not supported for tencent"): + litellm.image_edit( + model="tencent/hy-image-v3.5-preview", + image=PNG_BYTES, + prompt="turn the square into a paper lantern", + api_key="sk-tokenhub", + client=_client(captured), + ) + + assert captured == {} + + +@pytest.mark.parametrize( + ("form_params", "expected_body_params"), + [ + ({"watermark": "false", "seed": "7"}, {"watermark": False, "seed": 7}), + ({"watermark": "True", "seed": "-1"}, {"watermark": True, "seed": -1}), + ({"watermark": "maybe"}, {"watermark": "maybe"}), + ( + {"layer_decomposition": "true", "optimize_prompt_options[mode]": "fast"}, + {"layer_decomposition": True, "optimize_prompt_options": {"mode": "fast"}}, + ), + ( + {"sequential_image_generation_options[max_images]": "3", "user": "12345"}, + {"sequential_image_generation_options": {"max_images": 3}, "user": "12345"}, + ), + ], +) +def test_edit_parses_tokenhub_values_sent_as_form_strings( + form_params: dict[str, object], expected_body_params: dict[str, object] +) -> None: + captured: Final[dict[str, object]] = {} + + litellm.image_edit( + model="tencent/seedream-image-v5.0-flash", + image=PNG_BYTES, + prompt="turn the square into a paper lantern", + api_key="sk-tokenhub", + client=_client(captured), + **form_params, + ) + + assert captured["body"] == { + "model": "seedream-image-v5.0-flash", + "prompt": "turn the square into a paper lantern", + "images": [f"data:image/png;base64,{base64.b64encode(PNG_BYTES).decode()}"], + **expected_body_params, + } + + +@pytest.mark.parametrize( + ("params", "expected_body_params"), + [ + ({"background": "auto", "watermark": False}, {"watermark": False}), + ({"watermark": False, "seed": 7, "additional_drop_params": ["watermark"]}, {"seed": 7}), + ({"extra_body": {"watermark": False}, "additional_drop_params": ["watermark"]}, {}), + ], +) +def test_edit_keeps_dropped_params_out_of_the_tokenhub_body( + params: dict[str, object], expected_body_params: dict[str, object] +) -> None: + captured: Final[dict[str, object]] = {} + + litellm.image_edit( + model="tencent/seedream-image-v5.0-flash", + image=PNG_BYTES, + prompt="turn the square into a paper lantern", + api_key="sk-tokenhub", + client=_client(captured), + **params, + ) + + assert captured["body"] == { + "model": "seedream-image-v5.0-flash", + "prompt": "turn the square into a paper lantern", + "images": [f"data:image/png;base64,{base64.b64encode(PNG_BYTES).decode()}"], + **expected_body_params, + } + + +@pytest.mark.parametrize( + ("params", "expected_body_params"), + [ + ({"extra_body": {"response_format": "b64_json"}}, {"response_format": "b64_json"}), + ({"size": "1K", "extra_body": {"size": "2K"}}, {"size": "2K"}), + ({"extra_body": {"model": "seedream-image-v5.0-pro", "seed": 7}}, {"seed": 7}), + ({"extra_body": '{"seed": 7}'}, {}), + ], +) +def test_edit_sends_extra_body_fields_as_given( + params: dict[str, object], expected_body_params: dict[str, object] +) -> None: + captured: Final[dict[str, object]] = {} + + litellm.image_edit( + model="tencent/seedream-image-v5.0-flash", + image=PNG_BYTES, + prompt="turn the square into a paper lantern", + api_key="sk-tokenhub", + client=_client(captured), + **params, + ) + + assert captured["body"] == { + "model": "seedream-image-v5.0-flash", + "prompt": "turn the square into a paper lantern", + "images": [f"data:image/png;base64,{base64.b64encode(PNG_BYTES).decode()}"], + **expected_body_params, + } + + +def test_edit_forwards_tokenhub_only_params_and_extra_body() -> None: + captured: Final[dict[str, object]] = {} + + litellm.image_edit( + model="tencent/seedream-image-v5.0-flash", + image=PNG_BYTES, + prompt="turn the square into a paper lantern", + api_key="sk-tokenhub", + client=_client(captured), + user="end-user-42", + watermark=False, + seed=7, + extra_body={"sequential_image_generation": "disabled"}, + ) + + assert captured["body"] == { + "model": "seedream-image-v5.0-flash", + "prompt": "turn the square into a paper lantern", + "images": [f"data:image/png;base64,{base64.b64encode(PNG_BYTES).decode()}"], + "user": "end-user-42", + "watermark": False, + "seed": 7, + "sequential_image_generation": "disabled", + } diff --git a/tests/unit/llms/tencent/image_generation/__init__.py b/tests/unit/llms/tencent/image_generation/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/tencent/image_generation/test_tencent_image_generation_transformation.py b/tests/unit/llms/tencent/image_generation/test_tencent_image_generation_transformation.py new file mode 100644 index 00000000000..ab16382cdae --- /dev/null +++ b/tests/unit/llms/tencent/image_generation/test_tencent_image_generation_transformation.py @@ -0,0 +1,215 @@ +import json +from typing import Final + +import httpx +import openai +import pytest + +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler + +TOKENHUB_RESPONSE: Final[dict[str, object]] = { + "created": 1792551825, + "data": [{"output_format": "jpeg", "size": "1248x832", "url": "https://cos.example/lantern.jpg"}], + "model": "doubao-seedream-5-0-pro-260628", + "tokenhub_usage": {"total_tokens": 28125}, + "request_id": "c57b0374-b5fe-47db-b4b2-751315f920aa", +} + + +def _client(captured: dict[str, object], status_code: int = 200, body: dict[str, object] | None = None) -> HTTPHandler: + def respond(request: httpx.Request) -> httpx.Response: + captured["url"] = str(request.url) + captured["authorization"] = request.headers.get("authorization") + captured["body"] = json.loads(request.content) + return httpx.Response(status_code, json=TOKENHUB_RESPONSE if body is None else body) + + return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + + +def test_generation_posts_openai_and_tokenhub_params_to_the_wand_route() -> None: + captured: Final[dict[str, object]] = {} + + response: Final = litellm.image_generation( + model="tencent/seedream-image-v5.0-pro", + prompt="a red paper lantern", + size="2K", + response_format="url", + api_key="sk-tokenhub", + client=_client(captured), + images=["https://ref.example/lantern.png"], + watermark=False, + ) + + assert captured["url"] == "https://tokenhub-intl.tencentcloudmaas.com/v1/wand/si-image/generation" + assert captured["authorization"] == "Bearer sk-tokenhub" + assert captured["body"] == { + "model": "seedream-image-v5.0-pro", + "prompt": "a red paper lantern", + "size": "2K", + "response_format": "url", + "images": ["https://ref.example/lantern.png"], + "watermark": False, + } + assert response.data[0].url == "https://cos.example/lantern.jpg" + assert response.size == "1248x832" + assert response.output_format == "jpeg" + assert response.created == 1792551825 + assert response.usage.total_tokens == 28125 + assert response.usage.output_tokens == 28125 + + +def test_generation_returns_base64_images() -> None: + captured: Final[dict[str, object]] = {} + body: Final[dict[str, object]] = { + **TOKENHUB_RESPONSE, + "data": [{"output_format": "png", "size": "1024x1024", "b64_json": "aW1n"}], + } + + response: Final = litellm.image_generation( + model="tencent/seedream-image-v5.0-flash", + prompt="a red paper lantern", + response_format="b64_json", + api_key="sk-tokenhub", + client=_client(captured, body=body), + ) + + assert response.data[0].b64_json == "aW1n" + assert response.data[0].url is None + + +@pytest.mark.parametrize( + ("api_base", "expected_url"), + [ + ("https://tokenhub.example", "https://tokenhub.example/v1/wand/si-image/generation"), + ("https://tokenhub.example/v1", "https://tokenhub.example/v1/wand/si-image/generation"), + ("https://tokenhub.example/v1/", "https://tokenhub.example/v1/wand/si-image/generation"), + ], +) +def test_generation_resolves_a_configured_api_base_to_the_wand_route(api_base: str, expected_url: str) -> None: + captured: Final[dict[str, object]] = {} + + litellm.image_generation( + model="tencent/seedream-image-v5.0-lite", + prompt="a red paper lantern", + api_key="sk-tokenhub", + api_base=api_base, + client=_client(captured), + ) + + assert captured["url"] == expected_url + + +@pytest.mark.parametrize( + ("params", "expected_body_params"), + [ + ({"n": 1}, {}), + ({"n": 3, "drop_params": True}, {}), + ({"background": "auto"}, {}), + ({"background": "transparent", "output_format": "png"}, {"background": "transparent", "output_format": "png"}), + ({"user": "end-user-42"}, {"user": "end-user-42"}), + ], +) +def test_generation_sends_only_values_tokenhub_accepts( + params: dict[str, object], expected_body_params: dict[str, object] +) -> None: + captured: Final[dict[str, object]] = {} + + litellm.image_generation( + model="tencent/seedream-image-v5.0-pro", + prompt="a red paper lantern", + api_key="sk-tokenhub", + client=_client(captured), + **params, + ) + + assert captured["body"] == { + "model": "seedream-image-v5.0-pro", + "prompt": "a red paper lantern", + **expected_body_params, + } + + +def test_generation_rejects_more_than_one_image_without_drop_params() -> None: + captured: Final[dict[str, object]] = {} + + with pytest.raises(litellm.UnsupportedParamsError) as exc_info: + litellm.image_generation( + model="tencent/seedream-image-v5.0-pro", + prompt="a red paper lantern", + n=2, + api_key="sk-tokenhub", + client=_client(captured), + ) + + assert exc_info.value.status_code == 400 + assert captured == {} + + +@pytest.mark.parametrize( + ("error", "expected_message"), + [ + ( + { + "error": { + "message": "The model or service ID 'seedream-image-v5.0-pro' does not exist.", + "code": "400004", + "type": "gateway_error", + } + }, + "does not exist", + ), + ({"Type": 2, "Code": 1001, "Msg": "Invalid param", "Desc": ""}, "Invalid param"), + ], +) +def test_generation_surfaces_the_tokenhub_error_message(error: dict[str, object], expected_message: str) -> None: + with pytest.raises(litellm.BadRequestError) as exc_info: + litellm.image_generation( + model="tencent/seedream-image-v5.0-pro", + prompt="a red paper lantern", + api_key="sk-tokenhub", + client=_client({}, status_code=400, body=error), + ) + + assert expected_message in str(exc_info.value) + + +def test_generation_without_a_tokenhub_key_is_an_authentication_error(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("TENCENT_API_KEY", raising=False) + captured: Final[dict[str, object]] = {} + + with pytest.raises(litellm.AuthenticationError) as exc_info: + litellm.image_generation( + model="tencent/seedream-image-v5.0-pro", + prompt="a red paper lantern", + client=_client(captured), + ) + + assert exc_info.value.status_code == 401 + assert "TENCENT_API_KEY" in str(exc_info.value) + assert captured == {} + + +def test_generation_keeps_other_tencent_image_models_on_the_openai_compatible_route() -> None: + captured: Final[dict[str, object]] = {} + + def respond(request: httpx.Request) -> httpx.Response: + captured["url"] = str(request.url) + captured["body"] = json.loads(request.content) + return httpx.Response(200, json={"created": 1792551825, "data": [{"url": "https://cos.example/hy.png"}]}) + + response: Final = litellm.image_generation( + model="tencent/hy-image-v3.5-preview", + prompt="a red paper lantern", + quality="high", + api_key="sk-tokenhub", + client=openai.OpenAI( + api_key="sk-tokenhub", + base_url="https://tokenhub-intl.tencentcloudmaas.com/v1", + http_client=httpx.Client(transport=httpx.MockTransport(respond)), + ), + ) + + assert captured["url"] == "https://tokenhub-intl.tencentcloudmaas.com/v1/images/generations" + assert captured["body"] == {"model": "hy-image-v3.5-preview", "prompt": "a red paper lantern", "quality": "high"} + assert response.data[0].url == "https://cos.example/hy.png" diff --git a/tests/unit/llms/tencent/test_tencent_cost_calculator.py b/tests/unit/llms/tencent/test_tencent_cost_calculator.py new file mode 100644 index 00000000000..7f98ee02810 --- /dev/null +++ b/tests/unit/llms/tencent/test_tencent_cost_calculator.py @@ -0,0 +1,128 @@ +from typing import Final + +import httpx +import openai +import pytest + +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.types.utils import ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails + +SEEDREAM_MODELS: Final = ( + "tencent/seedream-image-v5.0-pro", + "tencent/seedream-image-v5.0-flash", + "tencent/seedream-image-v5.0-lite", +) + + +def _image_response(total_tokens: int, images: int = 1) -> ImageResponse: + return ImageResponse( + created=1792551825, + data=[ImageObject(url=f"https://cos.example/{index}.jpg") for index in range(images)], + usage=ImageUsage( + input_tokens=0, + input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=0), + output_tokens=total_tokens, + total_tokens=total_tokens, + ), + ) + + +@pytest.mark.parametrize("model", SEEDREAM_MODELS) +@pytest.mark.parametrize("call_type", ["image_generation", "image_edit"]) +def test_seedream_bills_tokenhub_usage_tokens_for_a_tier_size( + local_model_cost_map: None, model: str, call_type: str +) -> None: + token_rate: Final = litellm.model_cost[model]["output_cost_per_image_token"] + + cost: Final = litellm.completion_cost( + completion_response=_image_response(total_tokens=56250), + model=model, + call_type=call_type, + size="2K", + ) + + assert cost == pytest.approx(56250 * token_rate) + + +@pytest.mark.parametrize("model", SEEDREAM_MODELS) +def test_seedream_falls_back_to_the_per_image_price_without_usage(local_model_cost_map: None, model: str) -> None: + per_image_price: Final = litellm.model_cost[model]["output_cost_per_image"] + + cost: Final = litellm.completion_cost( + completion_response=_image_response(total_tokens=0, images=2), + model=model, + call_type="image_generation", + size="2K", + ) + + assert cost == pytest.approx(2 * per_image_price) + + +async def test_a_deployment_priced_tencent_image_model_outside_seedream_bills_its_deployment_price() -> None: + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"created": 1792551825, "data": [{"url": "https://cos.example/hy.png"}]}) + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "hy-image", + "litellm_params": {"model": "tencent/hy-image-v3.5-preview", "api_key": "sk-tokenhub"}, + "model_info": {"input_cost_per_image": 0.04}, + } + ] + ) + + response: Final = await router.aimage_generation( + model="hy-image", + prompt="a red paper lantern", + client=openai.AsyncOpenAI( + api_key="sk-tokenhub", + base_url="https://tokenhub-intl.tencentcloudmaas.com/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(respond)), + ), + ) + + assert response._hidden_params["response_cost"] == pytest.approx(0.04) + + +@pytest.mark.parametrize( + ("deployment_price", "expected_cost"), + [ + ({"output_cost_per_token": 0.000001}, 11250 * 0.000001), + ({"output_cost_per_image": 0.05}, 0.05), + ], +) +def test_a_seedream_deployment_priced_in_the_ui_bills_its_own_rate( + deployment_price: dict[str, float], expected_cost: float +) -> None: + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "created": 1792551825, + "data": [{"output_format": "jpeg", "size": "1024x1024", "url": "https://cos.example/0.jpg"}], + "tokenhub_usage": {"total_tokens": 11250}, + }, + ) + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "seedream-flash", + "litellm_params": { + "model": "tencent/seedream-image-v5.0-flash", + "api_key": "sk-tokenhub", + **deployment_price, + }, + } + ] + ) + + response: Final = router.image_generation( + model="seedream-flash", + prompt="a red paper lantern", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), + ) + + assert response._hidden_params["response_cost"] == pytest.approx(expected_cost) From 6063187fea3715087bddb7d23e7eaefaa5dea82d Mon Sep 17 00:00:00 2001 From: quietjules <349048734+quietjules@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:53:06 -0700 Subject: [PATCH 17/27] test(spend): audit #47506 billed background cost reaching the creator's loggers (#48919) Adds integration cells for a different key reading the response, message redaction, a rotated, deleted, or team-deleted creator, and a creator whose Langfuse keys resolve to nothing Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../test_background_response_billed_once.py | 183 ++++++++++++++++-- 1 file changed, 172 insertions(+), 11 deletions(-) diff --git a/tests/integration/spend/test_background_response_billed_once.py b/tests/integration/spend/test_background_response_billed_once.py index b207e46204a..5f948fac3ab 100644 --- a/tests/integration/spend/test_background_response_billed_once.py +++ b/tests/integration/spend/test_background_response_billed_once.py @@ -1,6 +1,7 @@ import asyncio import base64 import json +import math import uuid from collections.abc import Callable, Mapping from concurrent.futures import ThreadPoolExecutor @@ -10,9 +11,18 @@ from typing import Final import httpx import openai import pytest -from integration._support.client import Gateway, Scenario, eventually, held, list_value, object_value, string_value +from integration._support.client import ( + Gateway, + Scenario, + delete_key_if_present, + eventually, + held, + list_value, + object_value, + string_value, +) from integration._support.database import read_rows, write_rows -from integration._support.otlp_sink import decode_spans +from integration._support.otlp_sink import Span, decode_spans from integration._support.upstream import delete_scenario, register_scenario from integration._support.wire import Request, wire_server from integration.caching.response_cache_case import entry_key, redis_store @@ -859,7 +869,7 @@ LANGFUSE_COST: Final = "langfuse.observation.cost_details" _LANGFUSE_COST_DETAILS: Final = TypeAdapter(dict[str, float]) -def _langfuse_callback(host: str) -> dict[str, JsonValue]: +def _langfuse_callback(host: str, **callback_vars: JsonValue) -> dict[str, JsonValue]: return { "callback_name": "langfuse", "callback_type": "success", @@ -867,12 +877,17 @@ def _langfuse_callback(host: str) -> dict[str, JsonValue]: "langfuse_public_key": LANGFUSE_PUBLIC_KEY, "langfuse_secret_key": LANGFUSE_SECRET_KEY, "langfuse_host": host, + **callback_vars, }, } +def _logging_to(langfuse_url: str, **callback_vars: JsonValue) -> dict[str, JsonValue]: + return {"logging": [_langfuse_callback(langfuse_url, **callback_vars)]} + + def _key_with_its_own_logging(scenario: Scenario, langfuse_url: str) -> str: - return scenario.key(metadata={"logging": [_langfuse_callback(langfuse_url)]}) + return scenario.key(metadata=_logging_to(langfuse_url)) def _key_in_a_team_with_logging(scenario: Scenario, langfuse_url: str) -> str: @@ -892,17 +907,33 @@ def _trace_exports(langfuse: Recorder) -> tuple[Request, ...]: return tuple(request for request in langfuse.requests() if request.target.startswith(LANGFUSE_TRACES_PATH)) -def _exported_costs(langfuse: Recorder) -> tuple[float, ...]: - spans: Final = tuple( +def _exported_spans(langfuse: Recorder) -> tuple[Span, ...]: + return tuple( chain.from_iterable( decode_spans(export.body, export.headers.get("content-type", "")) for export in _trace_exports(langfuse) ) ) - return tuple( - _LANGFUSE_COST_DETAILS.validate_json(string_value(span["attributes"][LANGFUSE_COST]))["total"] - for span in spans - if LANGFUSE_COST in span["attributes"] - ) + + +def _span_cost(span: Span) -> float | None: + cost_details: Final = span["attributes"].get(LANGFUSE_COST) + return None if cost_details is None else _LANGFUSE_COST_DETAILS.validate_json(string_value(cost_details))["total"] + + +def _exported_costs(langfuse: Recorder) -> tuple[float, ...]: + return tuple(cost for cost in map(_span_cost, _exported_spans(langfuse)) if cost is not None) + + +def _priced_spans(langfuse: Recorder) -> tuple[Span, ...]: + return tuple(span for span in _exported_spans(langfuse) if (_span_cost(span) or 0.0) > 0) + + +def _billed_once(langfuse: Recorder, seconds: float) -> Span: + priced: Final = eventually(lambda: _priced_spans(langfuse), lambda spans: len(spans) > 0, seconds=seconds) + costs: Final = tuple(_span_cost(span) or 0.0 for span in priced) + assert len(costs) == 1, costs + assert math.isclose(costs[0], PRICE), costs + return priced[0] @pytest.mark.timeout(240) @@ -925,3 +956,133 @@ def test_the_billed_cost_reaches_the_langfuse_project_set_on_the_creating_key_or assert [cost for cost in exported if cost > 0] == [pytest.approx(PRICE)] credentials: Final = base64.b64encode(f"{LANGFUSE_PUBLIC_KEY}:{LANGFUSE_SECRET_KEY}".encode()).decode() assert {export.headers.get("authorization") for export in _trace_exports(langfuse)} == {f"Basic {credentials}"} + + +@pytest.mark.timeout(240) +def test_the_billed_cost_reaches_the_creating_keys_langfuse_project_and_never_the_readers(gateway: Gateway) -> None: + with ( + wire_server(langfuse_sink) as creator_wire, + wire_server(langfuse_sink) as reader_wire, + gateway.scenario() as scenario, + ): + creator_langfuse: Final = Recorder(creator_wire) + reader_langfuse: Final = Recorder(reader_wire) + team_id: Final = scenario.team() + user_id: Final = scenario.member(team_id) + creator: Final = scenario.key(team_id=team_id, user_id=user_id, metadata=_logging_to(creator_wire.url)) + reader: Final = scenario.key(team_id=team_id, user_id=user_id, metadata=_logging_to(reader_wire.url)) + upstream: Final = scripted_upstream(scenario, background_routes("completed")) + response_id: Final = create_background_response(scenario, creator, upstream.model) + reads: Final = tuple(read_response(gateway, reader, response_id) for _ in range(READS)) + assert reads in (settling_reads("completed", READS), free_reads("completed", READS)) + _billed_once(creator_langfuse, SPEND_LANDING_SECONDS) + eventually(lambda: _trace_exports(reader_langfuse), lambda exports: len(exports) > 0) + assert _priced_spans(reader_langfuse) == () + + +MESSAGE_LOGGING: Final[Mapping[str, Mapping[str, JsonValue]]] = { + "logged": {}, + "redacted": {"turn_off_message_logging": "true"}, +} + + +@pytest.mark.timeout(240) +@pytest.mark.parametrize("message_logging", MESSAGE_LOGGING.keys()) +def test_the_billed_observation_carries_the_response_text_only_when_the_creating_key_keeps_message_logging( + gateway: Gateway, message_logging: str +) -> None: + with wire_server(langfuse_sink) as wire, gateway.scenario() as scenario: + langfuse: Final = Recorder(wire) + key: Final = scenario.key(metadata=_logging_to(wire.url, **MESSAGE_LOGGING[message_logging])) + upstream: Final = scripted_upstream(scenario, background_routes("completed")) + response_id: Final = create_background_response(scenario, key, upstream.model) + reads: Final = tuple(read_response(gateway, key, response_id) for _ in range(READS)) + assert reads in (settling_reads("completed", READS), free_reads("completed", READS)) + billed: Final = _billed_once(langfuse, SPEND_LANDING_SECONDS) + assert ("pong" in json.dumps(billed["attributes"])) is (message_logging == "logged"), billed["attributes"] + + +def _generated_key(scenario: Scenario, **fields: JsonValue) -> str: + key: Final = string_value(scenario.gateway.post("/key/generate", fields)["key"]) + scenario.cleanups.callback(delete_key_if_present, scenario.gateway, key) + return key + + +def _delete_team_if_present(gateway: Gateway, team_id: str) -> None: + if gateway.rows('SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team_id,)): + gateway.post("/team/delete", {"team_ids": [team_id]}) + + +Retirement = Callable[[Scenario, str, str], tuple[str, Callable[[], object]]] + + +def _rotated_with_grace_then_relogged( + scenario: Scenario, billed_url: str, stale_url: str +) -> tuple[str, Callable[[], object]]: + key: Final = _generated_key(scenario, metadata=_logging_to(stale_url)) + + def rotate() -> None: + rotated: Final = string_value( + scenario.gateway.post("/key/regenerate", {"key": key, "grace_period": "1h"})["key"] + ) + scenario.cleanups.callback(delete_key_if_present, scenario.gateway, rotated) + scenario.gateway.post("/key/update", {"key": rotated, "metadata": _logging_to(billed_url)}) + + return key, rotate + + +def _deleted_key(scenario: Scenario, billed_url: str, _stale_url: str) -> tuple[str, Callable[[], object]]: + key: Final = _generated_key(scenario, metadata=_logging_to(billed_url)) + return key, lambda: scenario.gateway.post("/key/delete", {"keys": [key]}) + + +def _deleted_team(scenario: Scenario, billed_url: str, _stale_url: str) -> tuple[str, Callable[[], object]]: + team_id: Final = string_value( + scenario.gateway.post("/team/new", {"team_alias": f"integration-{uuid.uuid4().hex}"})["team_id"] + ) + scenario.cleanups.callback(_delete_team_if_present, scenario.gateway, team_id) + added: Final = scenario.gateway.request("POST", f"/team/{team_id}/callback", _langfuse_callback(billed_url)) + assert added.status_code == 200, added.text + key: Final = _generated_key(scenario, team_id=team_id) + return key, lambda: scenario.gateway.post("/team/delete", {"team_ids": [team_id]}) + + +RETIREMENTS: Final[Mapping[str, Retirement]] = { + "key_rotated_with_grace": _rotated_with_grace_then_relogged, + "key_deleted": _deleted_key, + "team_deleted": _deleted_team, +} + + +@pytest.mark.timeout(240) +@pytest.mark.parametrize("retired", RETIREMENTS.keys()) +def test_a_retired_creators_billed_cost_reaches_its_own_langfuse_project(gateway: Gateway, retired: str) -> None: + with ( + wire_server(langfuse_sink) as billed_wire, + wire_server(langfuse_sink) as stale_wire, + gateway.scenario() as scenario, + ): + billed_langfuse: Final = Recorder(billed_wire) + stale_langfuse: Final = Recorder(stale_wire) + key, retire = RETIREMENTS[retired](scenario, billed_wire.url, stale_wire.url) + upstream: Final = scripted_upstream(scenario, background_routes("queued")) + response_id: Final = create_background_response(scenario, key, upstream.model) + retire() + upstream.script(background_routes("completed")) + _billed_once(billed_langfuse, ONE_POLL_SECONDS + SPEND_LANDING_SECONDS) + assert [row["status"] for row in managed_rows(response_id)] == ["completed"] + assert _priced_spans(stale_langfuse) == () + + +@pytest.mark.timeout(240) +def test_a_creator_whose_langfuse_keys_resolve_to_nothing_is_still_billed_once(gateway: Gateway) -> None: + unset: Final = f"os.environ/INTEGRATION_UNSET_LANGFUSE_{uuid.uuid4().hex.upper()}" + with wire_server(langfuse_sink) as wire, gateway.scenario() as scenario: + key: Final = scenario.key(metadata=_logging_to(wire.url, langfuse_public_key=unset, langfuse_secret_key=unset)) + upstream: Final = scripted_upstream(scenario, background_routes("completed")) + response_id: Final = create_background_response(scenario, key, upstream.model) + reads: Final = tuple(read_response(gateway, key, response_id) for _ in range(READS)) + assert reads in (settling_reads("completed", READS), free_reads("completed", READS)) + assert_charged_exactly(lambda: key_spend(key), PRICE) + assert math.isclose(number(one_charged_row(upstream.model_id)["spend"]), PRICE) + assert [row["status"] for row in managed_rows(response_id)] == ["completed"] From 8bbce867d05b239593890118d31797c77060f8a4 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Wed, 28 Oct 2026 18:58:52 -0400 Subject: [PATCH 18/27] feat(proxy): accept request priority from the x-litellm-priority header (#47618) * feat(proxy): accept request priority from the x-litellm-priority header Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): only apply x-litellm-priority on scheduler-queued routes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): apply x-litellm-priority only where the router schedules the request The header now maps onto priority only for chat and text completions that a router serves, so the Cursor route's responses call and proxies started without a config never forward it to the provider. /queue/chat/completions now honors the header too, with the header winning over a body priority. * test(proxy): use a real ProxyLogging in the priority header routing test * fix(proxy): ignore a malformed priority header under drop_params instead of dropping the body priority * fix(proxy): treat an empty priority header as unset and pass a valid one on as an integer --------- Co-authored-by: Mubashir Osmani Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 12 +- litellm/proxy/litellm_pre_call_utils.py | 3 + litellm/proxy/proxy_server.py | 11 +- litellm/router.py | 15 +- litellm/router_utils/request_priority.py | 9 + .../router_code_coverage.py | 2 + .../proxy/proxy_server/test_routes_queue.py | 48 ++++++ .../proxy/test_common_request_processing.py | 163 ++++++++++++++++++ 8 files changed, 257 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index f5d6417fb79..70e305f46fb 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -266,6 +266,8 @@ from litellm.proxy.anthropic_endpoints.streaming_model_restamp import ( AnthropicStreamModelRestamper, ) from litellm.proxy.litellm_pre_call_utils import ( + ROUTER_SCHEDULED_ROUTE_TYPES, + SCHEDULER_PRIORITY_HEADER, LiteLLMProxyRequestSetup, add_litellm_data_to_request, get_metadata_variable_name, @@ -2478,14 +2480,20 @@ class ProxyBaseLLMRequestProcessing: self.data["model"] = alias_target aliased_model: Final = self.data.get("model") + if llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES and isinstance(aliased_model, str): + self.data.update( + llm_router.priority_header_update( + model=aliased_model, kwargs=self.data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER) + ) + ) litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id")) self.data["litellm_call_id"] = litellm_call_id bind_telemetry_call_id(litellm_call_id) - DDSpanTagger.tag_call_id(self.data.get("litellm_call_id")) + DDSpanTagger.tag_call_id(litellm_call_id) DDSpanTagger.tag_request( user_api_key_dict=user_api_key_dict, - requested_model=self.data.get("model"), + requested_model=aliased_model, ) ### AUTO STREAM USAGE TRACKING ### diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 24a31723faa..601d741c189 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -118,6 +118,9 @@ _SESSION_ID_VALUE_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") _SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$") +SCHEDULER_PRIORITY_HEADER: Final = "x-litellm-priority" +ROUTER_SCHEDULED_ROUTE_TYPES: Final = frozenset({"acompletion", "atext_completion"}) + # W3C Trace Context traceparent header: https://www.w3.org/TR/trace-context/ # e.g. "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" _TRACEPARENT_RE: Final = re.compile(r"^[0-9a-f]{2}-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$", re.IGNORECASE) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 64fb2791913..70e1c25a70a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -666,7 +666,11 @@ from litellm.proxy.list_api.common import ( problem_response, request_validation_problem, ) -from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_litellm_data_to_request +from litellm.proxy.litellm_pre_call_utils import ( + SCHEDULER_PRIORITY_HEADER, + LiteLLMProxyRequestSetup, + add_litellm_data_to_request, +) from litellm.proxy.logging_endpoints.callback_logs_endpoints import ( rust_control_plane_router, ) @@ -18481,6 +18485,11 @@ async def async_queue_request( raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}) router: Final = llm_router + data.update( + router.priority_header_update( + model=data["model"], kwargs=data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER) + ) + ) queued_priority, _ = router.split_request_priority(model=data["model"], kwargs=data) if "stream" in data and data["stream"] is True: # use generate_responses to stream responses diff --git a/litellm/router.py b/litellm/router.py index 3e6d82d9b07..5ca09499c3b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -359,6 +359,7 @@ from litellm.router_utils.request_priority import ( InvalidPriority, parse_default_priority, priority_error, + priority_header_fields, request_drops_params, resolve_request_priority, unqueueable_priority_message, @@ -5411,14 +5412,22 @@ class Router: request_priority: Final = resolve_request_priority( requested=kwargs.get("priority"), default_priority=self.default_priority, - drops_params=lambda: request_drops_params( - kwargs, self.default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs) - ), + drops_params=lambda: self._drops_params(model=model, kwargs=kwargs), ) if isinstance(request_priority, InvalidPriority): raise priority_error(model=model, message=request_priority.message) return request_priority, {key: value for key, value in kwargs.items() if key != "priority"} + def priority_header_update( + self, model: str, kwargs: Mapping[str, object], header: str | None + ) -> Mapping[str, int | str]: + return priority_header_fields(header, drops_params=lambda: self._drops_params(model=model, kwargs=kwargs)) + + def _drops_params(self, model: str, kwargs: Mapping[str, object]) -> bool: + return request_drops_params( + kwargs, self.default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs) + ) + async def _schedule_factory( self, model: str, diff --git a/litellm/router_utils/request_priority.py b/litellm/router_utils/request_priority.py index 14608cc1bcf..61627d4b0cf 100644 --- a/litellm/router_utils/request_priority.py +++ b/litellm/router_utils/request_priority.py @@ -47,6 +47,15 @@ def resolve_request_priority( return priority +def priority_header_fields(header: str | None, drops_params: Callable[[], bool]) -> Mapping[str, int | str]: + if not header: + return {} + priority: Final = _as_priority(header) + if isinstance(priority, InvalidPriority): + return {} if drops_params() else {"priority": header} + return {"priority": priority} + + def parse_default_priority(value: object) -> int | None | InvalidPriority: if value is None: return None diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index eee02a2f6b6..4c9afd26dad 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -175,6 +175,8 @@ ignored_function_names = [ "_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py "_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py "_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py + "priority_header_update", # Tested through the priority header tests in test_common_request_processing.py + "_drops_params", # Read by split_request_priority and priority_header_update, tested through both "_responses_event_announces_background_run", # Tested through the background Responses stream tests in test_router.py "_aresponses_background_stream_can_release", # Tested through the background Responses stream tests in test_router.py "_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py diff --git a/tests/unit/proxy/proxy_server/test_routes_queue.py b/tests/unit/proxy/proxy_server/test_routes_queue.py index a86314e3063..5cbf89f5e68 100644 --- a/tests/unit/proxy/proxy_server/test_routes_queue.py +++ b/tests/unit/proxy/proxy_server/test_routes_queue.py @@ -191,6 +191,54 @@ def test_queue_chat_completions_rejects_a_request_it_cannot_queue_as_an_invalid_ assert add_request.await_count == 0 +@pytest.mark.parametrize( + ("default_priority", "priority_header", "priority_fields", "expected_priority"), + [ + (5, "0", {"priority": 5}, 0), + (5, "0", {}, 0), + (None, "high", {"priority": 1, "drop_params": True}, 1), + (None, "", {"priority": 1}, 1), + ], + ids=("header-beats-body", "header-only", "drop-params-keeps-body", "empty-header-keeps-body"), +) +def test_queue_chat_completions_queues_at_the_priority_header( + client, + auth_as, + monkeypatch: pytest.MonkeyPatch, + default_priority: int | None, + priority_header: str, + priority_fields: dict[str, object], + expected_priority: int, +) -> None: + router: Final = _queue_router(monkeypatch, default_priority=default_priority) + payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], **priority_fields} + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + with auth_as(): + response: Final = client.post( + "/queue/chat/completions", json=payload, headers={"x-litellm-priority": priority_header} + ) + assert response.status_code == 200, response.text + assert [call.kwargs["request"].priority for call in add_request.await_args_list] == [expected_priority] + assert response.headers.get("x-litellm-priority") == str(expected_priority) + + +def test_queue_chat_completions_rejects_a_malformed_priority_header_like_a_malformed_body_value( + client, auth_as, monkeypatch: pytest.MonkeyPatch +) -> None: + router: Final = _queue_router(monkeypatch, default_priority=5) + payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], "priority": 1} + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + with auth_as(): + response: Final = client.post( + "/queue/chat/completions", json=payload, headers={"x-litellm-priority": "high"} + ) + assert response.status_code == 400, response.text + error: Final = response.json()["error"] + assert (error["type"], error["param"]) == ("invalid_request_error", "priority") + assert "priority must be an integer" in error["message"] + assert add_request.await_count == 0 + + def test_streamed_queue_responses_report_the_priority_the_router_queued_at( client, auth_as, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 790790459c3..fdcbcbfcd05 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -35,6 +35,7 @@ from litellm.constants import ( ) from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.opentelemetry import UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, ProxyConfig, @@ -513,6 +514,168 @@ class TestProxyBaseLLMRequestProcessing: assert exc_info.value.param == "model" add_litellm_data_to_request.assert_not_awaited() + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("path", "route_type", "has_router", "priority_header", "body_priority", "expected_priority"), + [ + ("/v1/chat/completions", "acompletion", True, "0", 5, 0), + ("/v1/completions", "atext_completion", True, "0", 5, 0), + ("/v1/chat/completions", "acompletion", True, "1", None, 1), + ("/v1/completions", "atext_completion", True, "1", None, 1), + ("/v1/chat/completions", "acompletion", True, None, 5, 5), + ("/v1/chat/completions", "acompletion", True, "", 5, 5), + ("/cursor/chat/completions", "aresponses", True, "0", None, None), + ("/v1/responses", "aresponses", True, "0", 5, 5), + ("/v1/embeddings", "aembedding", True, "0", None, None), + ("/v1/chat/completions", "acompletion", False, "0", None, None), + ("/v1/chat/completions", "acompletion", False, "0", 5, 5), + ], + ids=( + "chat-header-beats-body", + "text-completion-header-beats-body", + "chat-header-only", + "text-completion-header-only", + "chat-body-only", + "chat-empty-header-keeps-body", + "cursor-responses-route-ignores-header", + "responses-keeps-body", + "embeddings-ignores-header", + "routerless-ignores-header", + "routerless-keeps-body", + ), + ) + async def test_common_processing_pre_call_logic_applies_priority_header_only_where_the_router_schedules( + self, + path: str, + route_type: Literal["acompletion", "atext_completion", "aresponses", "aembedding"], + has_router: bool, + priority_header: str | None, + body_priority: int | None, + expected_priority: str | int | None, + ) -> None: + from litellm.proxy.proxy_server import ProxyConfig as RealProxyConfig + + header_entries: Final[tuple[tuple[bytes, bytes], ...]] = ( + () if priority_header is None else ((b"x-litellm-priority", priority_header.encode()),) + ) + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": path, + "scheme": "http", + "server": ("localhost", 4000), + "client": ("127.0.0.1", 12345), + "query_string": b"", + "headers": ((b"content-type", b"application/json"), *header_entries), + } + ) + router: Final = ( + Router( + model_list=[ + { + "model_name": "qa-model", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test", "mock_response": "hi"}, + } + ] + ) + if has_router + else None + ) + processing_obj: Final = ProxyBaseLLMRequestProcessing( + data={ + "model": "qa-model", + "messages": [{"role": "user", "content": "hi"}], + **({"priority": body_priority} if body_priority is not None else {}), + } + ) + returned_data, _ = await processing_obj.common_processing_pre_call_logic( + request=request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()), + proxy_config=RealProxyConfig(), + route_type=route_type, + llm_router=router, + ) + + assert ("priority" in returned_data, returned_data.get("priority")) == ( + expected_priority is not None, + expected_priority, + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("requested_model", "deployment_drops_params", "body_priority", "expected_priority"), + [ + ("qa-model", True, 5, 5), + ("qa-model", True, None, None), + ("key-alias", True, 5, 5), + ("qa-model", False, 5, "high"), + ], + ids=( + "drop-params-keeps-body", + "drop-params-without-body-leaves-default", + "drop-params-read-from-the-aliased-deployment", + "no-drop-params-passes-header-on-to-400", + ), + ) + async def test_common_processing_pre_call_logic_drops_only_a_malformed_priority_header_under_drop_params( + self, + requested_model: str, + deployment_drops_params: bool, + body_priority: int | None, + expected_priority: str | int | None, + ) -> None: + from litellm.proxy.proxy_server import ProxyConfig as RealProxyConfig + + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "scheme": "http", + "server": ("localhost", 4000), + "client": ("127.0.0.1", 12345), + "query_string": b"", + "headers": ((b"content-type", b"application/json"), (b"x-litellm-priority", b"high")), + } + ) + router: Final = Router( + model_list=[ + { + "model_name": "qa-model", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "sk-test", + "mock_response": "hi", + "drop_params": deployment_drops_params, + }, + } + ] + ) + processing_obj: Final = ProxyBaseLLMRequestProcessing( + data={ + "model": requested_model, + "messages": [{"role": "user", "content": "hi"}], + **({"priority": body_priority} if body_priority is not None else {}), + } + ) + returned_data, _ = await processing_obj.common_processing_pre_call_logic( + request=request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test", aliases={"key-alias": "qa-model"}), + proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()), + proxy_config=RealProxyConfig(), + route_type="acompletion", + llm_router=router, + ) + + assert ("priority" in returned_data, returned_data.get("priority")) == ( + expected_priority is not None, + expected_priority, + ) + @pytest.mark.asyncio @pytest.mark.parametrize("safe_memory_mode", [False, True]) @pytest.mark.parametrize( From 3f526a5017219f477fb24aa752511b9667ce51c0 Mon Sep 17 00:00:00 2001 From: agustin18 Date: Wed, 28 Oct 2026 20:03:03 -0300 Subject: [PATCH 19/27] fix(websearch): strip Claude Code instructional prefix in short-circuit search (#43458) * fix(websearch): strip Claude Code instructional prefix in short-circuit search Signed-off-by: agustin18 * fix(websearch): preserve quotes for unwrapped search queries Signed-off-by: agustin18 * test(websearch): replace mock with pure unit tests and cover empty queries Signed-off-by: agustin18 * style(websearch): fix ruff formatting blank line in handler Signed-off-by: agustin18 * refactor(websearch): enforce Final annotations, prevent rebinding, and retain e2e coverage Signed-off-by: agustin18 * test(websearch): assert server_tool_use block unconditionally Signed-off-by: agustin18 * fix(websearch): search only the query inside Claude Code's web search prompt Match Claude Code's exact "Perform a web search for the query:" prefix instead of a broad regex family, keep any quotes the user typed so exact-phrase searches still work, and test the search request at the HTTP boundary. --------- Signed-off-by: agustin18 Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../websearch_interception/handler.py | 5 +- .../test_websearch_short_circuit.py | 64 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index a38b8bed595..6e1ba3eea08 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -97,6 +97,8 @@ WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY: Final = "websearch_native_blocks" _RESPONSE_CONTENT_FIELD: Final = "content" +_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX: Final = "Perform a web search for the query:" + _ResponseT: Final = TypeVar("_ResponseT") @@ -363,7 +365,8 @@ class WebSearchInterceptionLogger(CustomLogger): get_last_user_message, ) - query: Final = get_last_user_message(cast(list[AllMessageValues], messages)) + last_user_message: Final = get_last_user_message(cast(list[AllMessageValues], messages)) or "" + query: Final = last_user_message.removeprefix(_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX).strip() if not query: return None diff --git a/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py b/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py index fdc3771e903..414bcc11602 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py @@ -6,10 +6,13 @@ and executes the search directly without routing through the backend LLM. """ import json +from typing import Final from unittest.mock import AsyncMock, patch import pytest +import respx +import litellm from litellm.integrations.websearch_interception.handler import ( WebSearchInterceptionLogger, ) @@ -169,6 +172,27 @@ class TestTryShortCircuitSearch: assert result is None + @pytest.mark.asyncio + @pytest.mark.parametrize( + "empty_prompt", + [ + " ", + "Perform a web search for the query: ", + ], + ) + async def test_does_not_short_circuit_empty_or_whitespace_user_message(self, empty_prompt: str): + """User message that is whitespace or empty query → returns None without executing search.""" + logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"]) + + result = await logger.try_short_circuit_search( + model="github_copilot/claude-sonnet-4", + messages=[{"role": "user", "content": empty_prompt}], + tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 8}], + custom_llm_provider="github_copilot", + ) + + assert result is None + @pytest.mark.asyncio async def test_search_failure_returns_error_text(self): """Search failure → response with error message, not exception""" @@ -226,6 +250,46 @@ class TestTryShortCircuitSearch: # Query extraction tests # --------------------------------------------------------------------------- +PERPLEXITY_SEARCH_URL: Final = "https://api.perplexity.ai/search" + + +@pytest.fixture +def perplexity_search(monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter) -> respx.Route: + monkeypatch.setenv("PERPLEXITYAI_API_KEY", "test-perplexity-key") + monkeypatch.delenv("PERPLEXITY_API_BASE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + return respx_mock.post(PERPLEXITY_SEARCH_URL).respond( + json={"results": [{"title": "Releases", "url": "https://example.com/releases", "snippet": "Release notes"}]} + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("user_message", "searched_query"), + [ + ("Perform a web search for the query: litellm latest release", "litellm latest release"), + ('Perform a web search for the query: "litellm proxy"', '"litellm proxy"'), + ], +) +async def test_short_circuit_searches_the_query_inside_claude_codes_prompt( + perplexity_search: respx.Route, user_message: str, searched_query: str +): + logger: Final = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + + result: Final = await logger.try_short_circuit_search( + model="bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=[{"role": "user", "content": user_message}], + tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 8}], + custom_llm_provider="bedrock", + ) + + assert perplexity_search.call_count == 1 + assert json.loads(perplexity_search.calls.last.request.content)["query"] == searched_query + assert result is not None + server_tool_use: Final = next(block for block in result["content"] if block["type"] == "server_tool_use") + assert server_tool_use["input"]["query"] == searched_query + # --------------------------------------------------------------------------- # Integration with entry point From d30c702c5cfe98d0f2c6832ce5ef5bb0ca8019de Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Wed, 28 Oct 2026 16:11:51 -0700 Subject: [PATCH 20/27] feat(accounting): add durable event ledger writer (#48883) * feat(accounting): add durable event ledger writer * fix(accounting): validate delegated transaction managers locally * docs(accounting): clarify ledger integration contracts * fix(accounting): return ledger conflicts as values and keep cleanup on the kind index Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(accounting): import assert_never from typing_extensions for Python 3.10 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(accounting): annotate ledger integration test locals as Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(accounting): finish ledger test conventions --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/db/accounting_ledger.py | 269 ++++++++++++ .../database/test_accounting_ledger.py | 412 +++++++++++++++++- tests/unit/proxy/db/test_accounting_ledger.py | 103 +++++ 3 files changed, 781 insertions(+), 3 deletions(-) create mode 100644 litellm/proxy/db/accounting_ledger.py create mode 100644 tests/unit/proxy/db/test_accounting_ledger.py diff --git a/litellm/proxy/db/accounting_ledger.py b/litellm/proxy/db/accounting_ledger.py new file mode 100644 index 00000000000..91f8307283e --- /dev/null +++ b/litellm/proxy/db/accounting_ledger.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +import hashlib +import json +from collections.abc import AsyncGenerator, Callable +from contextlib import AbstractAsyncContextManager, asynccontextmanager +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING, Final, Literal, TypeAlias + +from pydantic import AwareDatetime, ConfigDict, JsonValue, TypeAdapter +from typing_extensions import assert_never + +from litellm.accounting import CostEvent, DeliveryTarget, EventKind, InboundUsageEvent, OutboundUsageEvent +from litellm.proxy.db.db_span import db_span +from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper + +if TYPE_CHECKING: + from prisma import Prisma + + from litellm.proxy.utils import PrismaClient + +AccountingEvent: TypeAlias = InboundUsageEvent | OutboundUsageEvent | CostEvent +_JSON: Final = TypeAdapter(dict[str, JsonValue]) +_INSTANT: Final[TypeAdapter[datetime]] = TypeAdapter(AwareDatetime) +_STRINGS: Final = TypeAdapter(tuple[dict[str, str], ...]) +_TIMEOUT: Final = timedelta(seconds=10) + + +class AccountingConflictError(ValueError): + pass + + +_Transaction: TypeAlias = AbstractAsyncContextManager["Prisma"] + + +@dataclass(frozen=True, slots=True) +class _AcceptedEvent: + event_id: str + kind: EventKind + source_event_id: str | None + occurred_at: datetime + payload: str + payload_hash: str + targets: tuple[str, ...] + schema_version: int + version: int + + +def _json(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def _prepare( + event: AccountingEvent, + targets: tuple[DeliveryTarget, ...], + envelope: str, + schema_version: int, + version: int, +) -> _AcceptedEvent: + if type(schema_version) is not int or type(version) is not int or min(schema_version, version) < 1: + raise ValueError("Accounting versions must be positive integers") + validated: Final = type(event).model_validate_json(event.model_dump_json()) + payload: Final = _JSON.validate_json(envelope) + if payload.get("event") != validated.model_dump(mode="json"): + raise ValueError("Accounting envelope must contain the matching public event") + destinations: Final = tuple(sorted(frozenset(DeliveryTarget(target).value for target in targets))) + serialized: Final = _json(payload) + identity: Final = _json((schema_version, version, destinations, payload)) + kind: Final = ( + EventKind.COST + if isinstance(validated, CostEvent) + else EventKind.INBOUND_USAGE + if isinstance(validated, InboundUsageEvent) + else EventKind.OUTBOUND_USAGE + ) + return _AcceptedEvent( + event_id=validated.event_id, + kind=kind, + source_event_id=validated.usage_event_id if isinstance(validated, CostEvent) else None, + occurred_at=validated.occurred_at if isinstance(validated, CostEvent) else validated.ended_at, + payload=serialized, + payload_hash=hashlib.sha256(identity.encode()).hexdigest(), + targets=destinations, + schema_version=schema_version, + version=version, + ) + + +_INSERT_EVENT: Final = """ +INSERT INTO "LiteLLM_AccountingEvent" + (event_id, kind, source_event_id, schema_version, payload_hash, occurred_at, payload) +VALUES ($1, $2::"AccountingEventKind", $3, $4::int, $5, $6::timestamp, $7::jsonb) +ON CONFLICT (event_id) DO NOTHING RETURNING event_id +""" +_INSERT_DELIVERIES: Final = """ +INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version) +SELECT $1, value::"AccountingDeliveryTarget", $3::int FROM jsonb_array_elements_text($2::jsonb) +""" +_LOCK_SOURCE: Final = """ +SELECT event_id FROM "LiteLLM_AccountingEvent" +WHERE event_id = $1 AND kind = 'outbound_usage' FOR UPDATE +""" +_EXISTING_COST: Final = """ +SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE kind = 'cost' AND source_event_id = $1 +""" +_LOCK_EVENT: Final = """ +SELECT payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 FOR UPDATE +""" +_COMPLETED_FAMILY: Final = """ +root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage') +AND root.recorded_at < $1::timestamp +AND (root.kind = 'inbound_usage' OR EXISTS ( + SELECT 1 FROM "LiteLLM_AccountingEvent" AS cost WHERE cost.source_event_id = root.event_id AND cost.kind = 'cost' +)) +AND NOT EXISTS ( + SELECT 1 FROM "LiteLLM_AccountingEvent" AS member + WHERE (member.event_id = root.event_id OR (member.kind = 'cost' AND member.source_event_id = root.event_id)) + AND (member.recorded_at >= $1::timestamp + OR EXISTS (SELECT 1 FROM "LiteLLM_AccountingDelivery" AS delivery + WHERE delivery.event_id = member.event_id AND delivery.state != 'applied')) +) +""" +_RETENTION_CANDIDATES: Final = ( + 'SELECT root.event_id FROM "LiteLLM_AccountingEvent" AS root WHERE ' + + _COMPLETED_FAMILY + + " ORDER BY root.recorded_at, root.event_id LIMIT $2::int FOR UPDATE OF root SKIP LOCKED" +) +_ELIGIBLE_GROUP: Final = ( + 'SELECT root.event_id FROM "LiteLLM_AccountingEvent" AS root WHERE root.event_id = $2 AND ' + _COMPLETED_FAMILY +) +_DELETE_DELIVERIES: Final = """ +DELETE FROM "LiteLLM_AccountingDelivery" WHERE event_id IN ( + SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 OR (kind = 'cost' AND source_event_id = $1) +) +""" +_DELETE_EVENTS: Final = """ +DELETE FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 OR (kind = 'cost' AND source_event_id = $1) +""" + + +_Conflict: TypeAlias = Literal["missing_source", "competing_cost", "identity_conflict"] + + +async def _accept(db: Prisma, record: _AcceptedEvent) -> Literal["inserted", "duplicate"] | _Conflict: + if record.source_event_id is not None: + source: Final = _STRINGS.validate_python(await db.query_raw(_LOCK_SOURCE, record.source_event_id)) + if not source: + return "missing_source" + costs: Final = _STRINGS.validate_python(tuple(await db.query_raw(_EXISTING_COST, record.source_event_id))) + if any(cost["event_id"] != record.event_id for cost in costs): + return "competing_cost" + inserted: Final = _STRINGS.validate_python( + await db.query_raw( + _INSERT_EVENT, + record.event_id, + record.kind.value, + record.source_event_id, + record.schema_version, + record.payload_hash, + record.occurred_at.astimezone(timezone.utc).isoformat(), + record.payload, + ) + ) + if not inserted: + existing: Final = _STRINGS.validate_python(tuple(await db.query_raw(_LOCK_EVENT, record.event_id))) + if not existing or existing[0]["payload_hash"] != record.payload_hash: + return "identity_conflict" + return "duplicate" + await db.execute_raw(_INSERT_DELIVERIES, record.event_id, _json(record.targets), record.version) + return "inserted" + + +async def _delete_completed(db: Prisma, event_id: str, cutoff: str) -> int: + if not _STRINGS.validate_python(await db.query_raw(_ELIGIBLE_GROUP, cutoff, event_id)): + return 0 + await db.execute_raw(_DELETE_DELIVERIES, event_id) + return await db.execute_raw(_DELETE_EVENTS, event_id) + + +class AccountingLedger: + def __init__(self, transaction: Callable[[], _Transaction]) -> None: + self._transaction: Final = transaction + + @classmethod + def for_client(cls, client: PrismaClient) -> AccountingLedger: + from prisma.client import TransactionManager + + managers: Final = TypeAdapter(TransactionManager, config=ConfigDict(arbitrary_types_allowed=True)) + + @asynccontextmanager + async def transaction() -> AsyncGenerator[Prisma]: + primary: Final = writer_wrapper(client.db) + manager: Final = managers.validate_python(primary.tx(timeout=_TIMEOUT)) + async with db_span("accounting_ledger", "LiteLLM_AccountingEvent"), manager as db: + await db.execute_raw("SET LOCAL statement_timeout = 5000") + await db.execute_raw("SET LOCAL lock_timeout = 1000") + yield db + + return cls(transaction) + + async def insert_event( + self, + event: AccountingEvent, + targets: tuple[DeliveryTarget, ...], + *, + envelope: str, + schema_version: int = 1, + version: int = 1, + ) -> Literal["inserted", "duplicate"]: + """Accept T5/T6's serialized envelope, containing its public event under `event`. + + The immutable acceptance hash includes the target set and delivery version. + Changing either for an accepted event conflicts. T7 must provide a separate + add-target or new-version workflow that preserves receipts and prevents + duplicate contributions; redelivery cannot silently create or reset work. + + The Prisma adapter opens one interactive transaction per event, including + two SET LOCAL statements and a source-row lock for costs. I/O is async, + but awaiting this method inline adds commit latency per event, potentially + several times per request. T5 owns bounded asynchronous capture and an + explicit batching strategy; looping over this method does not batch commits. + Success acknowledges commit, independently of HTTP response acknowledgement. + """ + record: Final = _prepare(event, targets, envelope, schema_version, version) + async with self._transaction() as db: + result: Final = await _accept(db, record) + match result: + case "inserted" | "duplicate": + return result + case "missing_source": + raise AccountingConflictError("Cost source must be a durably accepted outbound usage event") + case "competing_cost": + raise AccountingConflictError("Usage event already has an authoritative cost") + case "identity_conflict": + raise AccountingConflictError("Event identity conflicts with its accepted content or delivery set") + case _: + assert_never(result) + + async def purge_completed( + self, + *, + redelivery_before: datetime | None = None, + recovery_before: datetime | None = None, + limit: int = 100, + ) -> int: + """Delete at most `limit` completed families once both retention owners permit it. + + T8 must supply a coherent recovery watermark including outstanding holds. + Omitting either watermark disables deletion. Each usage row and its cost + remain together so a retained charge never loses its source or vice versa. + Applied receipts are deleted with the family, ending its retained dedup + protection. The caller must guarantee events recorded before + redelivery_before cannot be redelivered through supported paths. + T8's recovery_before must also protect checkpoints, + outstanding holds and unfinished settlements; age alone is insufficient. + """ + if type(limit) is not int or not 1 <= limit <= 1000: + raise ValueError("Accounting retention batch size must be between 1 and 1000") + if redelivery_before is None or recovery_before is None: + return 0 + cutoff: Final = min( + _INSTANT.validate_python(redelivery_before).astimezone(timezone.utc), + _INSTANT.validate_python(recovery_before).astimezone(timezone.utc), + ).isoformat() + async with self._transaction() as db: + roots: Final = _STRINGS.validate_python(tuple(await db.query_raw(_RETENTION_CANDIDATES, cutoff, limit))) + deleted: Final = tuple([await _delete_completed(db, root["event_id"], cutoff) for root in roots]) + return sum(deleted) diff --git a/tests/integration/database/test_accounting_ledger.py b/tests/integration/database/test_accounting_ledger.py index 089ed872dee..0c2ceb62eaf 100644 --- a/tests/integration/database/test_accounting_ledger.py +++ b/tests/integration/database/test_accounting_ledger.py @@ -1,16 +1,48 @@ +from __future__ import annotations + +import asyncio +import json import os import subprocess import sys -from collections.abc import Iterator +from collections.abc import AsyncGenerator, AsyncIterator, Iterator, Mapping, Sequence +from contextlib import asynccontextmanager +from datetime import datetime, timezone +from decimal import Decimal +from itertools import chain from pathlib import Path -from typing import Final +from typing import TYPE_CHECKING, Final, TypeAlias import psycopg import pytest +import pytest_asyncio from psycopg import sql +from pydantic import JsonValue -from litellm.accounting import DeliveryState, DeliveryTarget, EventKind +from litellm.accounting import ( + CostEvent, + DeliveryState, + DeliveryTarget, + EventKind, + InboundUsageEvent, + Metric, + MetricName, + OutboundUsageEvent, + Outcome, + PricingSnapshot, +) +from litellm.proxy.db import accounting_ledger +from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingEvent, AccountingLedger +from litellm.types.utils import CallTypes from tests.integration._support.database import scratch_database +from tests.integration._support.prisma_clients import connected_proxy_prisma + +if TYPE_CHECKING: + from prisma import Prisma + + from litellm.proxy.utils import PrismaClient + +_Writer: TypeAlias = tuple["PrismaClient", AccountingLedger] _SCHEMA: Final = Path(__file__).resolve().parents[3] / "litellm-proxy-extras/litellm_proxy_extras/schema.prisma" _EVENT_INSERT: Final = ( @@ -126,3 +158,377 @@ def test_new_delivery_defaults_to_pending_with_no_attempts(ledger: psycopg.Conne (DeliveryTarget.KEY_SPEND.value,), ).fetchone() assert row == ("pending", 0) + + +_AT: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) +_CUTOFF: Final = datetime(2026, 2, 1, tzinfo=timezone.utc) +_TARGETS: Final = (DeliveryTarget.USER_SPEND, DeliveryTarget.TEAM_SPEND) + + +def _usage(event_id: str = "usage", *, inbound: bool = False) -> InboundUsageEvent | OutboundUsageEvent: + fields: Final = dict( + event_id=event_id, + request_id="request", + call_type=CallTypes.completion, + started_at=_AT, + ended_at=_AT, + outcome=Outcome.SUCCEEDED, + dimensions=(), + ) + if inbound: + return InboundUsageEvent(**fields, metrics=(Metric(name=MetricName.REQUESTS, value=Decimal(1)),)) + return OutboundUsageEvent(**fields, attempt_id=event_id, retry_of_attempt_id=None, metrics=None) + + +def _cost(source: str = "usage", event_id: str = "cost", amount: str = "0.125") -> CostEvent: + return CostEvent( + event_id=event_id, + usage_event_id=source, + occurred_at=_AT, + dimensions=(), + charges=(Metric(name=MetricName.SPEND, value=Decimal(amount), unit="USD"),), + snapshot=PricingSnapshot( + calculator_version="fixture", + model="fixture", + provider=None, + pricing_source="cost_map", + priced_model_key="fixture", + total_cost=Decimal(amount), + ), + ) + + +def _envelope(event: AccountingEvent, *, context: Mapping[str, JsonValue] | None = None) -> str: + return json.dumps({"event": event.model_dump(mode="json"), "context": context or {}}) + + +async def _insert( + store: AccountingLedger, + event: AccountingEvent, + targets: tuple[DeliveryTarget, ...] = _TARGETS, + *, + context: Mapping[str, JsonValue] | None = None, +) -> str: + return await store.insert_event(event, targets, envelope=_envelope(event, context=context)) + + +def _rows(database_url: str, statement: str, parameters: tuple[object, ...] = ()) -> Sequence[tuple[object, ...]]: + with psycopg.connect(database_url) as connection: + cursor: Final = connection.execute(statement, parameters) + return cursor.fetchall() if cursor.description is not None else [] + + +def _complete_and_age(database_url: str) -> None: + with psycopg.connect(database_url) as connection: + connection.execute('UPDATE "LiteLLM_AccountingEvent" SET recorded_at = %s', (_AT,)) + connection.execute("UPDATE \"LiteLLM_AccountingDelivery\" SET state = 'applied', applied_at = %s", (_AT,)) + + +@pytest_asyncio.fixture(loop_scope="function") +async def writer(ledger_database: str, monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[_Writer]: + monkeypatch.setenv("DATABASE_URL", ledger_database) + monkeypatch.delenv("DATABASE_URL_READ_REPLICA", raising=False) + _rows(ledger_database, 'TRUNCATE "LiteLLM_AccountingDelivery", "LiteLLM_AccountingEvent"') + async with connected_proxy_prisma(ledger_database) as client: + yield client, AccountingLedger.for_client(client) + + +@pytest.mark.asyncio +async def test_event_and_complete_delivery_set_commit_together(writer: _Writer, ledger_database: str) -> None: + _, store = writer + event: Final = _usage() + assert await _insert(store, event, context={"requested_region": "eu", "quantity": "1.25"}) == "inserted" + rows: Final = _rows(ledger_database, 'SELECT kind::text, source_event_id, payload FROM "LiteLLM_AccountingEvent"') + assert rows == [ + ("outbound_usage", None, json.loads(_envelope(event, context={"requested_region": "eu", "quantity": "1.25"}))) + ] + deliveries: Final = _rows( + ledger_database, 'SELECT target::text, state::text, attempts FROM "LiteLLM_AccountingDelivery"' + ) + assert set(deliveries) == {(target.value, "pending", 0) for target in _TARGETS} + + +@pytest.mark.asyncio +async def test_concurrent_duplicates_preserve_completed_deliveries(writer: _Writer, ledger_database: str) -> None: + _, store = writer + event: Final = _usage(inbound=True) + results: Final = await asyncio.gather(*(_insert(store, event) for _ in range(8))) + assert results.count("inserted") == 1 and results.count("duplicate") == 7 + _complete_and_age(ledger_database) + before: Final = _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY target') + reordered: Final = json.dumps(json.loads(_envelope(event)), sort_keys=True, indent=2) + assert await store.insert_event(event, tuple(reversed(_TARGETS)) + _TARGETS, envelope=reordered) == "duplicate" + assert _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY target') == before + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(1,)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("change", ["payload", "context", "targets", "schema_version", "version"]) +async def test_conflicting_acceptance_cannot_overwrite_or_add_work( + writer: _Writer, ledger_database: str, change: str +) -> None: + _, store = writer + event: Final = _usage() + await _insert(store, event) + altered: Final = event.model_copy(update={"outcome": Outcome.FAILED}) if change == "payload" else event + targets: Final = (DeliveryTarget.KEY_SPEND,) if change == "targets" else _TARGETS + context: Final = {"override": "0.5"} if change == "context" else {} + options: Final = {change: 2} if change in ("schema_version", "version") else {} + with pytest.raises(AccountingConflictError, match="conflicts"): + await store.insert_event(altered, targets, envelope=_envelope(altered, context=context), **options) + assert _rows(ledger_database, 'SELECT payload FROM "LiteLLM_AccountingEvent"') == [(json.loads(_envelope(event)),)] + assert set(_rows(ledger_database, 'SELECT target::text FROM "LiteLLM_AccountingDelivery"')) == { + (t.value,) for t in _TARGETS + } + + +@pytest.mark.asyncio +async def test_competing_cost_identities_accept_only_one_authoritative_charge( + writer: _Writer, ledger_database: str +) -> None: + _, store = writer + await _insert(store, _usage()) + results: Final = await asyncio.gather( + _insert(store, _cost()), _insert(store, _cost(event_id="other-cost")), return_exceptions=True + ) + assert sum(result == "inserted" for result in results) == 1 + assert sum(isinstance(result, AccountingConflictError) for result in results) == 1 + accepted: Final = _rows( + ledger_database, "SELECT event_id, payload FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'" + ) + assert len(accepted) == 1 + event: Final = CostEvent.model_validate_json(json.dumps(accepted[0][1]["event"])) + assert await _insert(store, event) == "duplicate" + assert event.snapshot.total_cost == Decimal("0.125") + with pytest.raises(AccountingConflictError): + await _insert(store, _cost(event_id=event.event_id, amount="0.5")) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("source", ["missing", "inbound", "cost"]) +async def test_cost_requires_an_existing_outbound_source(writer: _Writer, ledger_database: str, source: str) -> None: + _, store = writer + if source == "inbound": + await _insert(store, _usage("source", inbound=True)) + if source == "cost": + await _insert(store, _usage()) + await _insert(store, _cost(event_id="source")) + with pytest.raises(AccountingConflictError, match="outbound usage"): + await _insert(store, _cost(source="source", event_id="invalid")) + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE event_id = 'invalid'") == [ + (0,) + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["before_commit", "after_commit", "cancellation"]) +async def test_interruption_rolls_back_or_retry_recognizes_commit( + writer: _Writer, ledger_database: str, failure: str +) -> None: + client, store = writer + + @asynccontextmanager + async def interrupted() -> AsyncGenerator[Prisma]: + async with client.writer_db.tx() as tx: + yield tx + if failure == "before_commit": + raise RuntimeError("injected pre-commit failure") + if failure == "cancellation": + raise asyncio.CancelledError + raise RuntimeError("injected lost acknowledgement") + + error: Final = asyncio.CancelledError if failure == "cancellation" else RuntimeError + with pytest.raises(error): + await _insert(AccountingLedger(interrupted), _usage()) + expected: Final = 1 if failure == "after_commit" else 0 + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(expected,)] + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(expected * len(_TARGETS),)] + assert await _insert(store, _usage()) == ("duplicate" if expected else "inserted") + + +@pytest.mark.asyncio +async def test_delivery_insert_failure_rolls_back_event(writer: _Writer, ledger_database: str) -> None: + from prisma.errors import RawQueryError + + _, store = writer + with psycopg.connect(ledger_database) as connection: + connection.execute( + 'ALTER TABLE "LiteLLM_AccountingDelivery" ADD CONSTRAINT reject_delivery CHECK (version < 1)' + ) + try: + with pytest.raises(RawQueryError, match="reject_delivery"): + await _insert(store, _usage()) + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(0,)] + finally: + with psycopg.connect(ledger_database) as connection: + connection.execute('ALTER TABLE "LiteLLM_AccountingDelivery" DROP CONSTRAINT reject_delivery') + + +@pytest.mark.asyncio +@pytest.mark.parametrize("blocked", ["pending", "parked", "unpriced", "recent_cost", "recovery"]) +async def test_retention_preserves_incomplete_or_recovery_needed_families( + writer: _Writer, ledger_database: str, blocked: str +) -> None: + _, store = writer + await _insert(store, _usage()) + if blocked != "unpriced": + await _insert(store, _cost()) + _complete_and_age(ledger_database) + with psycopg.connect(ledger_database) as connection: + if blocked in ("pending", "parked"): + connection.execute( + "UPDATE \"LiteLLM_AccountingDelivery\" SET state = %s WHERE event_id = 'cost'", (blocked,) + ) + if blocked == "recent_cost": + connection.execute( + "UPDATE \"LiteLLM_AccountingEvent\" SET recorded_at = %s WHERE event_id = 'cost'", (_CUTOFF,) + ) + before: Final = _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent" ORDER BY event_id') + assert ( + await store.purge_completed( + redelivery_before=_CUTOFF, recovery_before=_AT if blocked == "recovery" else _CUTOFF + ) + == 0 + ) + assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent" ORDER BY event_id') == before + + +@pytest.mark.asyncio +async def test_retention_removes_completed_family_and_receipts_together(writer: _Writer, ledger_database: str) -> None: + _, store = writer + await _insert(store, _usage()) + await _insert(store, _cost()) + _complete_and_age(ledger_database) + assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF) == 2 + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(0,)] + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(0,)] + + +@pytest.mark.asyncio +async def test_retention_respects_batch_bound_and_skips_locked_families(writer: _Writer, ledger_database: str) -> None: + client, store = writer + for name in ("a", "b", "c"): + await _insert(store, _usage(name, inbound=True)) + _complete_and_age(ledger_database) + async with client.writer_db.tx() as tx: + await tx.query_raw("SELECT event_id FROM \"LiteLLM_AccountingEvent\" WHERE event_id = 'a' FOR UPDATE") + assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF, limit=1) == 1 + assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent" ORDER BY event_id') == [ + ("a",), + ("c",), + ] + assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF) == 2 + + +@pytest.mark.asyncio +async def test_retention_makes_progress_past_pending_and_unpriced_events(writer: _Writer, ledger_database: str) -> None: + _, store = writer + await _insert(store, _usage("a", inbound=True)) + await _insert(store, _usage("b")) + await _insert(store, _usage("c", inbound=True)) + _complete_and_age(ledger_database) + _rows(ledger_database, "UPDATE \"LiteLLM_AccountingDelivery\" SET state = 'pending' WHERE event_id = 'a'") + assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF, limit=1) == 1 + assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent" ORDER BY event_id') == [ + ("a",), + ("b",), + ] + + +@pytest.mark.asyncio +async def test_zero_cost_and_empty_delivery_sets_are_durable(writer: _Writer, ledger_database: str) -> None: + _, store = writer + await _insert(store, _usage(), targets=()) + event: Final = _cost(amount="0") + assert await _insert(store, event, targets=()) == "inserted" + assert await _insert(store, event, targets=()) == "duplicate" + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(0,)] + _complete_and_age(ledger_database) + assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF) == 2 + + +@pytest.mark.asyncio +async def test_retention_cannot_delete_source_while_cost_acceptance_is_uncommitted( + writer: _Writer, ledger_database: str +) -> None: + client, store = writer + await _insert(store, _usage()) + _complete_and_age(ledger_database) + inserted: Final = asyncio.Event() + release: Final = asyncio.Event() + + @asynccontextmanager + async def paused_commit() -> AsyncGenerator[Prisma]: + async with client.writer_db.tx() as tx: + yield tx + inserted.set() + await release.wait() + + task: Final = asyncio.create_task(_insert(AccountingLedger(paused_commit), _cost())) + try: + await asyncio.wait_for(inserted.wait(), timeout=5) + assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF) == 0 + finally: + release.set() + await task + assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent" ORDER BY event_id') == [ + ("cost",), + ("usage",), + ] + + +@pytest.mark.asyncio +async def test_retention_failure_rolls_back_event_and_receipt_deletion(writer: _Writer, ledger_database: str) -> None: + client, store = writer + await _insert(store, _usage(inbound=True)) + _complete_and_age(ledger_database) + + @asynccontextmanager + async def interrupted() -> AsyncGenerator[Prisma]: + async with client.writer_db.tx() as tx: + yield tx + raise RuntimeError("injected cleanup failure") + + with pytest.raises(RuntimeError, match="cleanup failure"): + await AccountingLedger(interrupted).purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF) + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(1,)] + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(len(_TARGETS),)] + assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF) == 1 + + +@pytest.mark.asyncio +async def test_ledger_reads_and_writes_use_primary_when_replica_is_configured( + writer: _Writer, ledger_database: str, monkeypatch: pytest.MonkeyPatch +) -> None: + with scratch_database("replica") as replica_url: + monkeypatch.setenv("DATABASE_URL_READ_REPLICA", replica_url) + async with connected_proxy_prisma(ledger_database) as client: + store: Final = AccountingLedger.for_client(client) + await _insert(store, _usage()) + assert await _insert(store, _usage()) == "duplicate" + assert await _insert(store, _cost()) == "inserted" + _complete_and_age(ledger_database) + assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF) == 2 + assert _rows(replica_url, "SELECT to_regclass('public.\"LiteLLM_AccountingEvent\"')") == [(None,)] + + +def _index_conditions(plan: object) -> tuple[tuple[object, object], ...]: + if isinstance(plan, dict): + own: Final = ((plan.get("Index Name"), plan.get("Index Cond")),) if "Index Name" in plan else () + children: Final = (plan.get("Plan"), *plan.get("Plans", ())) + return own + tuple(chain.from_iterable(_index_conditions(child) for child in children if child is not None)) + return tuple(chain.from_iterable(_index_conditions(item) for item in plan)) if isinstance(plan, list) else () + + +@pytest.mark.parametrize("statement", [accounting_ledger._DELETE_EVENTS, accounting_ledger._DELETE_DELIVERIES]) +def test_family_cleanup_lookups_use_the_kind_source_index(ledger_database: str, statement: str) -> None: + explained: Final = "EXPLAIN (FORMAT JSON) " + statement.replace("$1", "%s") + with psycopg.connect(ledger_database) as connection: + connection.execute("SET enable_seqscan = off") + plan: Final = connection.execute(explained, ("usage",) * statement.count("$1")).fetchone() + assert plan is not None + conditions: Final = _index_conditions(plan[0]) + assert any( + name == "LiteLLM_AccountingEvent_kind_source_event_id_idx" and "source_event_id" in str(condition) + for name, condition in conditions + ), conditions diff --git a/tests/unit/proxy/db/test_accounting_ledger.py b/tests/unit/proxy/db/test_accounting_ledger.py new file mode 100644 index 00000000000..3916fe869e6 --- /dev/null +++ b/tests/unit/proxy/db/test_accounting_ledger.py @@ -0,0 +1,103 @@ +import json +from datetime import datetime, timezone +from decimal import Decimal +from typing import Final, Literal, NoReturn + +import pytest + +from litellm.accounting import InboundUsageEvent, Metric, MetricName, Outcome +from litellm.proxy.db.accounting_ledger import AccountingLedger +from litellm.types.utils import CallTypes + + +def _no_transaction() -> NoReturn: + raise AssertionError("Invalid accounting input must not open a database transaction") + + +@pytest.fixture +def event() -> InboundUsageEvent: + return InboundUsageEvent( + event_id="caller", + request_id="caller", + call_type=CallTypes.completion, + started_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + ended_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + outcome=Outcome.SUCCEEDED, + dimensions=(), + metrics=(Metric(name=MetricName.REQUESTS, value=Decimal(1)),), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("envelope", ["{}", "[]", '{"event":null}', '{"event":{}}', "not json"]) +async def test_invalid_envelopes_fail_before_database_access(event: InboundUsageEvent, envelope: str) -> None: + with pytest.raises(ValueError, match=r"Accounting envelope|validation error"): + await AccountingLedger(_no_transaction).insert_event(event, (), envelope=envelope) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("version", [0, -1, True, 1.5]) +@pytest.mark.parametrize("field", ["version", "schema_version"]) +async def test_invalid_versions_fail_before_database_access( + event: InboundUsageEvent, version: int | float, field: Literal["version", "schema_version"] +) -> None: + with pytest.raises(ValueError, match="positive integers"): + await AccountingLedger(_no_transaction).insert_event( + event, + (), + envelope="{}", + **{field: version}, # pyright: ignore[reportArgumentType] # Exercise invalid runtime versions + ) + + +@pytest.mark.asyncio +async def test_constructed_invalid_events_are_revalidated(event: InboundUsageEvent) -> None: + invalid: Final = event.model_copy(update={"event_id": ""}) + with pytest.raises(ValueError, match="event_id"): + await AccountingLedger(_no_transaction).insert_event(invalid, (), envelope="{}") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("limit", [0, -1, 1001, True, 1.5]) +async def test_retention_rejects_invalid_batch_bounds(limit: int | float) -> None: + with pytest.raises(ValueError, match="batch size"): + await AccountingLedger(_no_transaction).purge_completed( + limit=limit # pyright: ignore[reportArgumentType] # Exercise invalid runtime batch sizes + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("missing", ["both", "redelivery", "recovery"]) +async def test_retention_requires_both_safety_watermarks(missing: Literal["both", "redelivery", "recovery"]) -> None: + cutoff: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + deleted: Final = await AccountingLedger(_no_transaction).purge_completed( + redelivery_before=cutoff if missing == "recovery" else None, + recovery_before=cutoff if missing == "redelivery" else None, + ) + assert deleted == 0 + + +@pytest.mark.asyncio +async def test_retention_rejects_ambiguous_naive_watermarks() -> None: + with pytest.raises(ValueError, match="timezone"): + await AccountingLedger(_no_transaction).purge_completed( + redelivery_before=datetime(2026, 1, 1), recovery_before=datetime(2026, 1, 1, tzinfo=timezone.utc) + ) + + +@pytest.mark.asyncio +async def test_nonfinite_context_is_rejected_before_database_access(event: InboundUsageEvent) -> None: + envelope: Final = json.dumps({"event": event.model_dump(mode="json"), "context": {"price": float("nan")}}) + with pytest.raises(ValueError, match="Out of range float"): + await AccountingLedger(_no_transaction).insert_event(event, (), envelope=envelope) + + +@pytest.mark.asyncio +async def test_unknown_delivery_target_is_rejected_before_database_access(event: InboundUsageEvent) -> None: + envelope: Final = json.dumps({"event": event.model_dump(mode="json")}) + with pytest.raises(ValueError, match="not a valid DeliveryTarget"): + await AccountingLedger(_no_transaction).insert_event( + event, + ("unknown",), # pyright: ignore[reportArgumentType] # Exercise an unknown runtime target + envelope=envelope, + ) From f89b9c61504801fcc99e40e37af532bb38bbfa30 Mon Sep 17 00:00:00 2001 From: rj-cairn <349020452+rj-cairn@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:12:10 -0700 Subject: [PATCH 21/27] refactor(types): replace Any with proven types in 28 files (#48903) * refactor(types): replace Any with proven types in 52 files Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(types): use typing_extensions.TypedDict for validated seams Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(types): revert strict Anthropic response validation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(types): accept a null finish reason from Bedrock Cohere and TwelveLabs Pegasus * test(bedrock): annotate the Pegasus null finish reason test --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 4 +- litellm/litellm_core_utils/safe_json_dumps.py | 32 +++++++-------- ...mazon_twelvelabs_pegasus_transformation.py | 15 ++++++- .../base_invoke_transformation.py | 39 +++++++++++++++---- .../llms/bedrock/realtime/transformation.py | 3 +- .../llms/dashscope/embed/transformation.py | 30 +++++++++++--- litellm/llms/databricks/streaming_utils.py | 9 ++++- litellm/llms/gigachat/authenticator.py | 15 ++++++- litellm/llms/openai/realtime/handler.py | 7 +++- .../perplexity/embedding/transformation.py | 33 ++++++++++++---- .../vertex_gemma_models/transformation.py | 22 ++++++++--- .../volcengine/responses/transformation.py | 5 ++- litellm/proxy/client/models.py | 9 +++-- .../guardrail_hooks/openai/moderations.py | 2 +- litellm/proxy/litellm_pre_call_utils.py | 12 +++--- .../auto_router_endpoints.py | 5 ++- .../router_settings_endpoints.py | 4 +- .../management_helpers/team_roster_sync.py | 3 +- .../mcp_server/discoverable_endpoints.py | 3 +- .../pass_through_endpoints.py | 24 ++++++++---- .../spend_tracking/key_metadata_recovery.py | 5 ++- .../secret_managers/aws_secret_manager_v2.py | 23 ++++++++--- .../integrations/rag/bedrock_knowledgebase.py | 10 ++--- litellm/types/llms/bedrock_invoke_agents.py | 10 ++--- litellm/types/llms/gemini.py | 11 +++--- litellm/types/llms/langgraph.py | 11 +++--- litellm/types/mcp_server/tool_registry.py | 10 ++--- .../guardrails/guardrail_hooks/enkryptai.py | 10 ++--- .../test_base_invoke_transformation.py | 27 +++++++++++++ .../test_twelvelabs_pegasus_transformation.py | 28 +++++++++++++ 30 files changed, 308 insertions(+), 113 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 80f1f8f5eca..8b66c27b930 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3481,7 +3481,7 @@ class Logging(LiteLLMLoggingBaseClass): start_time=start_time, end_time=end_time, print_verbose=print_verbose, - level=LogfireLevel.INFO.value, + level=LogfireLevel.INFO, ) if callback == "lunary" and lunaryLogger is not None: @@ -4442,7 +4442,7 @@ class Logging(LiteLLMLoggingBaseClass): response_obj=result, start_time=start_time, end_time=end_time, - level=LogfireLevel.ERROR.value, + level=LogfireLevel.ERROR, print_verbose=print_verbose, ) diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index 63242a580e7..24c401281ef 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -34,7 +34,7 @@ def safe_json_structure( def _transform(key: str | None, value: str) -> str: return value if value_transform is None else value_transform(key, value) - def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> Any: + def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> object: # Check for maximum depth. if depth > max_depth: return "MaxDepthExceeded" @@ -48,32 +48,32 @@ def safe_json_structure( if id(obj) in seen: return "CircularReference Detected" seen.add(id(obj)) - result: dict | list | tuple | set | str if isinstance(obj, dict): - result = {} - for k, v in obj.items(): - if isinstance(k, (str)): - clean_k = k.replace("\x00", "") if "\x00" in k else k - result[clean_k] = _serialize(v, seen, depth + 1, clean_k) + mapping_result: Final = { + (clean_k := k.replace("\x00", "") if "\x00" in k else k): _serialize(v, seen, depth + 1, clean_k) + for k, v in obj.items() + if isinstance(k, str) + } seen.remove(id(obj)) - return result + return mapping_result elif isinstance(obj, list): - result = [_serialize(item, seen, depth + 1, key) for item in obj] + list_result: Final = [_serialize(item, seen, depth + 1, key) for item in obj] seen.remove(id(obj)) - return result + return list_result elif isinstance(obj, tuple): - result = tuple(_serialize(item, seen, depth + 1, key) for item in obj) + tuple_result: Final = tuple(_serialize(item, seen, depth + 1, key) for item in obj) seen.remove(id(obj)) - return result + return tuple_result elif isinstance(obj, set): - result = sorted([_serialize(item, seen, depth + 1, key) for item in obj]) + sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj] + set_result: Final = sorted(sortable_values) seen.remove(id(obj)) - return result + return set_result elif isinstance(obj, BaseModel): dumped: Final = obj.model_dump() - result = _serialize(dumped, seen, depth + 1, key) + model_result: Final = _serialize(dumped, seen, depth + 1, key) seen.remove(id(obj)) - return result + return model_result else: # Fall back to string conversion for non-serializable objects. try: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py index f7566ac4923..dc14ef005b5 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py @@ -10,6 +10,8 @@ import time from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -34,6 +36,15 @@ else: LiteLLMLoggingObj = Any +@with_config(ConfigDict(extra="allow", strict=True)) +class _PegasusResponse(TypedDict): + message: ReadOnly[NotRequired[str | None]] + finishReason: ReadOnly[NotRequired[str | None]] + + +_PEGASUS_RESPONSE: Final = TypeAdapter(_PegasusResponse) + + class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): """ Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for @@ -207,7 +218,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): ModelResponse with choices[0].message.content and finish_reason """ try: - completion_response: Final = raw_response.json() + completion_response: Final = _PEGASUS_RESPONSE.validate_python(raw_response.json()) except Exception as e: raise BedrockError( message=f"Error parsing response: {raw_response.text}, error: {e}", @@ -224,7 +235,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): message_content: Final = completion_response.get("message", "") # Extract finish reason and map to LiteLLM format - finish_reason_raw: Final = completion_response.get("finishReason", "stop") + finish_reason_raw: Final = completion_response.get("finishReason") or "stop" finish_reason: Final = map_finish_reason(finish_reason_raw) # Set the response content diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 51b5a99272e..852179e8caa 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -1,11 +1,12 @@ import copy import json import time -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, cast, get_args import httpx -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -48,6 +49,27 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM _GUARDRAIL_CONFIG_VALIDATOR: Final["TypeAdapter[GuardrailConfigBlock]"] = TypeAdapter(GuardrailConfigBlock) + +@with_config(ConfigDict(extra="allow", strict=True)) +class _CohereGeneration(TypedDict): + text: ReadOnly[str | None] + finish_reason: ReadOnly[str | None] + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _CohereInvokeResponse(TypedDict): + text: NotRequired[ReadOnly[str | None]] + generations: NotRequired[ReadOnly[Sequence[_CohereGeneration]]] + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _LlamaInvokeResponse(TypedDict): + generation: ReadOnly[str | None] + + +_COHERE_INVOKE_RESPONSE: Final = TypeAdapter(_CohereInvokeResponse) +_LLAMA_INVOKE_RESPONSE: Final = TypeAdapter(_LlamaInvokeResponse) + _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = ( "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}" ) @@ -310,12 +332,13 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): outputText: str | None = None try: if provider == "cohere": - if "text" in completion_response: - outputText = completion_response["text"] - elif "generations" in completion_response: - outputText = completion_response["generations"][0]["text"] + cohere_response: Final = _COHERE_INVOKE_RESPONSE.validate_python(completion_response) + if "text" in cohere_response: + outputText = cohere_response["text"] + elif "generations" in cohere_response: + outputText = cohere_response["generations"][0]["text"] model_response.choices[0].finish_reason = map_finish_reason( - completion_response["generations"][0]["finish_reason"] + cohere_response["generations"][0]["finish_reason"] or "stop" ) elif provider == "anthropic": return litellm.AmazonAnthropicClaudeConfig().transform_response( @@ -361,7 +384,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): elif provider == "ai21": outputText = completion_response.get("completions")[0].get("data").get("text") elif provider == "meta" or provider == "llama" or provider == "deepseek_r1": - outputText = completion_response["generation"] + outputText = _LLAMA_INVOKE_RESPONSE.validate_python(completion_response)["generation"] elif provider == "mistral": outputText = litellm.AmazonMistralConfig.get_outputText(completion_response, model_response) else: # amazon titan diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index 9cb38822881..952d8858f78 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -53,6 +53,7 @@ from litellm.types.realtime import ( RealtimeResponseTypedDict, ) +_JSON_OBJECT_MESSAGE: Final = TypeAdapter(dict[str, JsonValue], config=ConfigDict(strict=True)) _JSON_VALUE: Final = TypeAdapter(object) _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) @@ -856,7 +857,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): List of Bedrock format messages (JSON strings) """ try: - json_message: Final[dict[str, JsonValue]] = json.loads(message) + json_message: Final = _JSON_OBJECT_MESSAGE.validate_python(json.loads(message)) except json.JSONDecodeError: verbose_logger.warning("Invalid JSON message: %s", message[:200]) return [] diff --git a/litellm/llms/dashscope/embed/transformation.py b/litellm/llms/dashscope/embed/transformation.py index d0c2b4dff0c..35132346261 100644 --- a/litellm/llms/dashscope/embed/transformation.py +++ b/litellm/llms/dashscope/embed/transformation.py @@ -14,6 +14,8 @@ Docs - https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api from typing import Final import httpx +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import ensure_api_route @@ -28,6 +30,23 @@ from ..common_utils import DashScopeError DEFAULT_API_BASE: Final = "https://dashscope.aliyuncs.com/compatible-mode/v1" +@with_config(ConfigDict(extra="allow", strict=True)) +class _DashScopeUsage(TypedDict, total=False): + prompt_tokens: NotRequired[ReadOnly[int]] + total_tokens: NotRequired[ReadOnly[int]] + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _DashScopeEmbeddingResponse(TypedDict, total=False): + data: NotRequired[ReadOnly[list[object]]] + model: NotRequired[ReadOnly[str | None]] + usage: NotRequired[ReadOnly[_DashScopeUsage | None]] + id: NotRequired[ReadOnly[object]] + + +_DASHSCOPE_EMBEDDING_RESPONSE: Final = TypeAdapter(_DashScopeEmbeddingResponse) + + class DashScopeEmbeddingConfig(BaseEmbeddingConfig): """ Reference: https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api @@ -152,11 +171,12 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig): message=message, ) + parsed_response: Final = _DASHSCOPE_EMBEDDING_RESPONSE.validate_python(response_json) model_response.object = "list" - model_response.data = response_json.get("data", []) - model_response.model = response_json.get("model", model) + model_response.data = parsed_response.get("data", []) + model_response.model = parsed_response.get("model", model) - usage: Final = response_json.get("usage") or {} + usage: Final = parsed_response.get("usage") or {} prompt_tokens: Final = usage.get("prompt_tokens", 0) total_tokens: Final = usage.get("total_tokens", prompt_tokens) setattr( @@ -169,8 +189,8 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig): ), ) - if "id" in response_json: - setattr(model_response, "id", response_json["id"]) + if "id" in parsed_response: + setattr(model_response, "id", parsed_response["id"]) return model_response diff --git a/litellm/llms/databricks/streaming_utils.py b/litellm/llms/databricks/streaming_utils.py index eca87f9dd5f..cc85012b803 100644 --- a/litellm/llms/databricks/streaming_utils.py +++ b/litellm/llms/databricks/streaming_utils.py @@ -1,6 +1,8 @@ import json from typing import Final +from pydantic import ConfigDict, TypeAdapter + import litellm from litellm import verbose_logger from litellm.types.llms.openai import ( @@ -23,6 +25,9 @@ def _usage_block(usage: Usage) -> ChatCompletionUsageBlock: ) +_JSON_CHUNK: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) + + class ModelResponseIterator: def __init__(self, streaming_response, sync_stream: bool): self.streaming_response = streaming_response @@ -99,7 +104,7 @@ class ModelResponseIterator: self.response_iterator = self.streaming_response chunk: Final = self._next_sse_data() try: - json_chunk: Final = json.loads(chunk) + json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk)) return self.chunk_parser(chunk=json_chunk) except ValueError as e: verbose_logger.debug( @@ -126,7 +131,7 @@ class ModelResponseIterator: if chunk == "[DONE]": raise StopAsyncIteration try: - json_chunk: Final = json.loads(chunk) + json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk)) return self.chunk_parser(chunk=json_chunk) except ValueError as e: verbose_logger.debug( diff --git a/litellm/llms/gigachat/authenticator.py b/litellm/llms/gigachat/authenticator.py index d43e203d7a1..f37c6e88896 100644 --- a/litellm/llms/gigachat/authenticator.py +++ b/litellm/llms/gigachat/authenticator.py @@ -12,6 +12,8 @@ from types import MappingProxyType from typing import Final import httpx +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_logger from litellm.caching.caching import InMemoryCache @@ -37,6 +39,17 @@ _token_cache: Final = InMemoryCache() _NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) +@with_config(ConfigDict(extra="allow", strict=True)) +class _GigaChatTokenResponse(TypedDict): + tok: NotRequired[ReadOnly[str | None]] + access_token: NotRequired[ReadOnly[str | None]] + exp: NotRequired[ReadOnly[int | float | str | None]] + expires_at: NotRequired[ReadOnly[int | float | str | None]] + + +_TOKEN_RESPONSE: Final = TypeAdapter(_GigaChatTokenResponse) + + class GigaChatAuthError(BaseLLMException): """GigaChat authentication error.""" @@ -230,7 +243,7 @@ async def _request_token_async( def _parse_token_response(response: httpx.Response) -> tuple[str, int]: """Parse OAuth token response.""" - data: Final = response.json() + data: Final = _TOKEN_RESPONSE.validate_python(response.json()) # GigaChat returns either 'tok'/'exp' or 'access_token'/'expires_at' access_token: Final = data.get("tok") or data.get("access_token") diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 080fc3d24c9..b7e4cc6868f 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -5,7 +5,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. """ import ssl -from typing import Any, Final, cast +from typing import TYPE_CHECKING, Final, cast from litellm._logging import redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES @@ -23,6 +23,9 @@ from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from ..common_utils import is_openai_backed_api_base from ..openai import OpenAIChatCompletion +if TYPE_CHECKING: + from fastapi import WebSocket + class OpenAIRealtime(OpenAIChatCompletion): """ @@ -131,7 +134,7 @@ class OpenAIRealtime(OpenAIChatCompletion): async def async_realtime( self, model: str, - websocket: Any, + websocket: "WebSocket", logging_obj: LiteLLMLogging, api_base: str | None = None, api_key: str | None = None, diff --git a/litellm/llms/perplexity/embedding/transformation.py b/litellm/llms/perplexity/embedding/transformation.py index 7b4da933976..147d344795d 100644 --- a/litellm/llms/perplexity/embedding/transformation.py +++ b/litellm/llms/perplexity/embedding/transformation.py @@ -12,12 +12,14 @@ This module decodes them into float arrays for OpenAI-compatible responses. """ import base64 +import builtins import struct -from collections.abc import Mapping -from typing import Final +from collections.abc import Mapping, Sequence +from typing import Final, Literal import httpx -from pydantic import ConfigDict, TypeAdapter +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import ReadOnly, TypedDict from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import join_api_base @@ -31,6 +33,17 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i _INT: Final = TypeAdapter(int) +@with_config(ConfigDict(extra="allow", strict=True)) +class _PerplexityEmbeddingResponse(TypedDict, total=False): + model: ReadOnly[str | None] + object: ReadOnly[Literal["list"]] + data: ReadOnly[Sequence[builtins.object] | Mapping[str, builtins.object] | Literal[""]] + usage: ReadOnly[builtins.object] + + +_EMBEDDING_RESPONSE_ADAPTER: Final = TypeAdapter(_PerplexityEmbeddingResponse) + + class PerplexityEmbeddingError(BaseLLMException): def __init__( self, @@ -153,14 +166,18 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig): litellm_params: dict = {}, ) -> EmbeddingResponse: try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final[object] = raw_response.json() except Exception: raise PerplexityEmbeddingError(message=raw_response.text, status_code=raw_response.status_code) - model_response.model = raw_response_json.get("model", model) - model_response.object = raw_response_json.get("object", "list") + response_data: Final[_PerplexityEmbeddingResponse] = _EMBEDDING_RESPONSE_ADAPTER.validate_python( + raw_response_json + ) - raw_data: Final = raw_response_json.get("data", []) + model_response.model = response_data.get("model", model) + model_response.object = response_data.get("object", "list") + + raw_data: Final = response_data.get("data", []) decoded_data: Final[list[dict[str, object]]] = [] for item in raw_data: decoded_item = dict(_JSON_OBJECT.validate_python(item)) @@ -168,7 +185,7 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig): decoded_data.append(decoded_item) model_response.data = decoded_data - usage_data: Final = _JSON_OBJECT.validate_python(raw_response_json.get("usage", {})) + usage_data: Final = _JSON_OBJECT.validate_python(response_data.get("usage", {})) usage: Final = Usage( prompt_tokens=_INT.validate_python( usage_data.get("prompt_tokens", 0) or usage_data.get("total_tokens", 0) or 0 diff --git a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py index 9fa86ec30f5..ec2fd16f93e 100644 --- a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py +++ b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py @@ -9,10 +9,11 @@ The actual message transformation reuses OpenAIGPTConfig since Gemma uses OpenAI """ from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Final, cast import httpx -from pydantic import ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError +from typing_extensions import NotRequired, ReadOnly, TypeAliasType, TypedDict from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import ( @@ -32,6 +33,17 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +class _PredictionsEnvelope(TypedDict): + predictions: NotRequired[ReadOnly[dict[str, object]]] + + +_PredictionsEnvelopeType = TypeAliasType("_PredictionsEnvelopeType", _PredictionsEnvelope) +_PREDICTIONS_ENVELOPE: Final[TypeAdapter[_PredictionsEnvelope]] = TypeAdapter( + _PredictionsEnvelopeType, + config=ConfigDict(strict=True, extra="allow"), +) + + def parse_vertex_gemma_container_error(predictions: object) -> VertexGemmaContainerError | None: try: return VertexGemmaContainerError.model_validate(predictions) @@ -142,7 +154,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): def _unwrap_predictions_response( self, - response_json: dict[str, Any], + response_json: _PredictionsEnvelope, ) -> dict[str, object]: """ Unwrap the Vertex Gemma predictions format to OpenAI format. @@ -354,7 +366,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): message=f"Request failed: {response.text}", ) - response_json: Final = response.json() + response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json()) # Unwrap predictions to get OpenAI-compatible response openai_response: Final = self._unwrap_predictions_response(response_json) @@ -449,7 +461,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): message=f"Request failed: {response.text}", ) - response_json: Final = response.json() + response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json()) # Unwrap predictions to get OpenAI-compatible response openai_response: Final = self._unwrap_predictions_response(response_json) diff --git a/litellm/llms/volcengine/responses/transformation.py b/litellm/llms/volcengine/responses/transformation.py index 747ea21e4fb..4076ed06fc2 100644 --- a/litellm/llms/volcengine/responses/transformation.py +++ b/litellm/llms/volcengine/responses/transformation.py @@ -3,6 +3,7 @@ from types import UnionType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, Union, get_args, get_origin import httpx +from pydantic import ConfigDict, TypeAdapter from pydantic import fields as pyd_fields import litellm @@ -37,6 +38,8 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_RESPONSE_BODY: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) + class _EventModelClass(Protocol): @property @@ -420,7 +423,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): @staticmethod def _parsed_response_body(raw_response: httpx.Response) -> dict[str, object]: - return raw_response.json() + return _RESPONSE_BODY.validate_python(raw_response.json()) @staticmethod def _annotation_origin(annotation: object) -> object: diff --git a/litellm/proxy/client/models.py b/litellm/proxy/client/models.py index 1361a3e6364..859322f061d 100644 --- a/litellm/proxy/client/models.py +++ b/litellm/proxy/client/models.py @@ -3,9 +3,12 @@ from collections.abc import Mapping from typing import Any, Final import requests +from pydantic import ConfigDict, TypeAdapter from .exceptions import NotFoundError, UnauthorizedError +_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) + class ModelsManagementClient: def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30): @@ -109,7 +112,7 @@ class ModelsManagementClient: try: response: Final = session.send(request.prepare(), timeout=self._timeout) response.raise_for_status() - return response.json() + return _JSON_OBJECT.validate_python(response.json()) except requests.exceptions.HTTPError as e: if e.response.status_code == 401: raise UnauthorizedError(e) @@ -145,7 +148,7 @@ class ModelsManagementClient: try: response: Final = session.send(request.prepare(), timeout=self._timeout) response.raise_for_status() - return response.json() + return _JSON_OBJECT.validate_python(response.json()) except requests.exceptions.HTTPError as e: if e.response.status_code == 401: raise UnauthorizedError(e) @@ -287,7 +290,7 @@ class ModelsManagementClient: try: response: Final = session.send(request.prepare(), timeout=self._timeout) response.raise_for_status() - return response.json() + return _JSON_OBJECT.validate_python(response.json()) except requests.exceptions.HTTPError as e: if e.response.status_code == 401: raise UnauthorizedError(e) diff --git a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py index 98be35079c3..16b3c45f6f8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py +++ b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py @@ -169,7 +169,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): from litellm.types.llms.openai import OpenAIModerationResponse - return OpenAIModerationResponse(**response.json()) + return OpenAIModerationResponse.model_validate(response.json()) def _check_moderation_result(self, moderation_response: "OpenAIModerationResponse") -> None: """ diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 601d741c189..d4528bc85e8 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -691,7 +691,7 @@ def _metadata_string_tags(metadata_objects: Iterable[Mapping[str, object]]) -> t def _strip_untrusted_request_header_controls( - headers: Any, + headers: object, *, allow_client_message_redaction_opt_out: bool = False, ) -> None: @@ -1695,7 +1695,7 @@ class LiteLLMProxyRequestSetup: @staticmethod def add_internal_user_from_user_mapping( - general_settings: dict | None, + general_settings: Mapping[str, object] | None, user_api_key_dict: UserAPIKeyAuth, headers: dict, ) -> UserAPIKeyAuth: @@ -1714,7 +1714,7 @@ class LiteLLMProxyRequestSetup: return user_api_key_dict @staticmethod - def get_user_from_headers(headers: dict, general_settings: dict | None = None) -> str | None: + def get_user_from_headers(headers: dict, general_settings: Mapping[str, object] | None = None) -> str | None: """ Get the user from the specified header if `general_settings.user_header_name` is set. """ @@ -1735,7 +1735,9 @@ class LiteLLMProxyRequestSetup: return user @staticmethod - def get_openai_org_id_from_headers(headers: dict, general_settings: dict | None = None) -> str | None: + def get_openai_org_id_from_headers( + headers: dict, general_settings: Mapping[str, object] | None = None + ) -> str | None: """ Get the OpenAI Org ID from the headers. """ @@ -1826,7 +1828,7 @@ class LiteLLMProxyRequestSetup: headers: dict, request_data: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth, - general_settings: dict[str, object] | None = None, + general_settings: Mapping[str, object] | None = None, ) -> LitellmDataForBackendLLMCall: """ - Adds user from headers diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 7a843aecc0c..4f48f3c57b9 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -231,9 +231,12 @@ def _shadow_eval_attempts(prisma_client: "PrismaClient") -> _ShadowEvalAttemptTa return _ShadowEvalAttemptRepository(prisma_client).table +_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True)) + + async def _query_raw(prisma_client: "PrismaClient", query: str, *args: object) -> Sequence[Mapping[str, object]]: async with db_span("auto_router_report_query", sql_relation(query)): - return await prisma_client.db.query_raw(query, *args) + return _RAW_ROWS.validate_python(await prisma_client.db.query_raw(query, *args)) async def _execute_raw(prisma_client: "PrismaClient", query: str, *args: object) -> int: diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py index 1f4ca442d20..41c4b5cfaf7 100644 --- a/litellm/proxy/management_endpoints/router_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py @@ -10,7 +10,7 @@ GET /router/fields - Get router settings field definitions without values (for U import inspect from collections.abc import Mapping from types import MappingProxyType -from typing import Any, Final, get_args +from typing import Final, get_args from fastapi import APIRouter, Depends from pydantic import Field @@ -32,7 +32,7 @@ router: Final = APIRouter() class RouterSettingsResponse(LiteLLMBaseModel): fields: list[RouterSettingsField] = Field(description="List of all configurable router settings with metadata") - current_values: dict[str, Any] = Field(description="Current values of router settings") + current_values: dict[str, object] = Field(description="Current values of router settings") routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option") routing_group_strategies: tuple[str, ...] = Field( description="Strategies supported when constructing a routing group" diff --git a/litellm/proxy/management_helpers/team_roster_sync.py b/litellm/proxy/management_helpers/team_roster_sync.py index dc0ba2e47d4..08be455a55d 100644 --- a/litellm/proxy/management_helpers/team_roster_sync.py +++ b/litellm/proxy/management_helpers/team_roster_sync.py @@ -40,6 +40,7 @@ from litellm.proxy.management_helpers.access_group_team_sync import ( ) from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.chunked_in import delete_many_in, find_many_in, update_many_in +from litellm.repositories.team_repository import TeamRepository if TYPE_CHECKING: from prisma import Prisma @@ -144,7 +145,7 @@ def _changes_nothing(plan: RosterPlan) -> bool: async def _unchanged_roster(prisma_client: PrismaClient, team_id: str) -> RosterSync | TeamGone: - team_row: Final = await prisma_client.writer_db.litellm_teamtable.find_unique(where={"team_id": team_id}) + team_row: Final = await TeamRepository(prisma_client, use_writer=True).table.find_unique(where={"team_id": team_id}) if team_row is None: return TeamGone(team_id=team_id) team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump()) diff --git a/litellm/proxy/mcp_server/discoverable_endpoints.py b/litellm/proxy/mcp_server/discoverable_endpoints.py index eb12b15cdae..07a267d6caa 100644 --- a/litellm/proxy/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/mcp_server/discoverable_endpoints.py @@ -300,12 +300,13 @@ def decode_state_hash(encrypted_state: str) -> dict: if decrypted_json is None: raise ValueError("Failed to decrypt state parameter") - state_data: Final = json.loads(decrypted_json) + state_data: Final = _DECODED_STATE.validate_python(json.loads(decrypted_json)) return state_data _MANAGED_AUTH_CODE_PREFIX: Final = "llm_scope_" _DECODED_JSON: Final = TypeAdapter(object) +_DECODED_STATE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) class _ManagedAuthorizationCode(LiteLLMBaseModel): diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 650dcae3239..e017eee3d27 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -30,6 +30,7 @@ from pydantic import TypeAdapter from starlette.datastructures import UploadFile as StarletteUploadFile from starlette.routing import BaseRoute, Route from starlette.websockets import WebSocketState +from typing_extensions import ReadOnly from websockets.asyncio.client import connect from websockets.exceptions import ( ConnectionClosedError, @@ -173,8 +174,17 @@ def _is_mergeable_client_metadata(key: str, value: object) -> bool: return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping)) +class _RegisteredPassThroughRoute(TypedDict): + endpoint_id: ReadOnly[str] + path: ReadOnly[str | None] + type: ReadOnly[str] + methods: ReadOnly[str | bool | list[str] | tuple[str, ...] | Mapping[str, object]] + auth: ReadOnly[bool] + passthrough_params: ReadOnly[Mapping[str, object]] + + # Global registry to track registered pass-through routes and prevent memory leaks -_registered_pass_through_routes: Final[dict[str, dict[str, str | bool | list[str] | Mapping[str, object]]]] = {} +_registered_pass_through_routes: Final[dict[str, _RegisteredPassThroughRoute]] = {} def get_response_body(response: httpx.Response) -> dict | None: @@ -2239,7 +2249,7 @@ def create_pass_through_route( status_code=status.HTTP_405_METHOD_NOT_ALLOWED, detail=f"Method {request.method} is not allowed for pass-through endpoint {path}.", ) - target_params: Final = { + target_params: Final[dict[str, object]] = { "target": target, "custom_headers": custom_headers, "forward_headers": _forward_headers, @@ -2271,7 +2281,9 @@ def create_pass_through_route( # Ensure custom_headers is a dict. Botocore returns a HeadersDict # for SigV4-prepared requests, which is a Mapping but not a dict. - headers_dict: Final = dict(param_custom_headers) if isinstance(param_custom_headers, Mapping) else {} + headers_dict: Final[dict[str, object]] = ( + {**param_custom_headers} if isinstance(param_custom_headers, Mapping) else {} + ) # Ensure query_params and custom_body are dicts or None final_query_params: Final = query_params_data if isinstance(query_params_data, dict) else {} @@ -3104,9 +3116,7 @@ class SafeRouteAdder: return f"{path}/{{subpath:path}}" return None - def registered_route_methods( - route_info: Mapping[str, str | bool | Sequence[str] | Mapping[str, object]], - ) -> tuple[tuple[str, str], ...]: + def registered_route_methods(route_info: _RegisteredPassThroughRoute) -> tuple[tuple[str, str], ...]: starlette_path: Final = registered_route_path(route_info) methods: Final = route_info.get("methods") if starlette_path is None or not isinstance(methods, (list, tuple)): @@ -3462,7 +3472,7 @@ class InitPassThroughEndpointHelpers: return (comparison_route, *forwarded) @staticmethod - def get_registered_pass_through_route(route: str, method: str | None = None) -> dict[str, Any] | None: + def get_registered_pass_through_route(route: str, method: str | None = None) -> _RegisteredPassThroughRoute | None: """Get passthrough params for a given route and optionally filter by HTTP method""" comparison_route: Final = InitPassThroughEndpointHelpers._route_for_registry_lookup(route) for key in _registered_pass_through_routes: diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index 5442c0f3f81..8fd4fbfcb8b 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -6,7 +6,7 @@ from datetime import datetime, timedelta from types import MappingProxyType from typing import Final, Literal, TypeVar -from pydantic import TypeAdapter +from pydantic import ConfigDict, TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger @@ -155,6 +155,7 @@ class _DailyUserSpendOwnerRow(LiteLLMBaseModel): last_owner: str | None = None +_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True)) _TOKEN_DIGEST_ROWS: Final = TypeAdapter(tuple[_TokenDigestRow, ...]) _SPEND_LOG_DIGEST_ROWS: Final = TypeAdapter(tuple[_SpendLogDigestRow, ...]) _DAILY_USER_SPEND_OWNER_ROWS: Final = TypeAdapter(tuple[_DailyUserSpendOwnerRow, ...]) @@ -194,7 +195,7 @@ async def _rows_within_the_statement_timeout( await transaction.execute_raw(_SPEND_LOG_STATEMENT_TIMEOUT_SQL) for setting in planner_settings: await transaction.execute_raw(setting) - return await transaction.query_raw(sql, *params) + return _RAW_ROWS.validate_python(await transaction.query_raw(sql, *params)) async def _reverse_hash_key_metadata( diff --git a/litellm/secret_managers/aws_secret_manager_v2.py b/litellm/secret_managers/aws_secret_manager_v2.py index 98d12e497da..de8566e6fdc 100644 --- a/litellm/secret_managers/aws_secret_manager_v2.py +++ b/litellm/secret_managers/aws_secret_manager_v2.py @@ -21,6 +21,8 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -41,6 +43,15 @@ if TYPE_CHECKING: from botocore.awsrequest import HTTPHeaders +@with_config(ConfigDict(extra="allow", strict=True)) +class _SecretStringResponse(TypedDict): + SecretString: ReadOnly[str] + + +_SECRET_STRING_RESPONSE: Final = TypeAdapter(_SecretStringResponse) +_AWS_SECRET_MANAGER_RESPONSE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) + + class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): def __init__( self, @@ -154,7 +165,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): try: response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8")) response.raise_for_status() - return response.json()["SecretString"] + return _SECRET_STRING_RESPONSE.validate_python(response.json())["SecretString"] except httpx.TimeoutException: raise ValueError("Timeout error occurred") except Exception as e: @@ -204,7 +215,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): try: response: Final = sync_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8")) - return response.json()["SecretString"] + return _SECRET_STRING_RESPONSE.validate_python(response.json())["SecretString"] except httpx.TimeoutException: raise ValueError("Timeout error occurred") except httpx.HTTPStatusError as e: @@ -424,7 +435,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): try: response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8")) response.raise_for_status() - return response.json() + return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json()) except httpx.HTTPStatusError as err: raise ValueError(f"HTTP error occurred: {err.response.text}") except httpx.TimeoutException: @@ -481,7 +492,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): try: response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8")) response.raise_for_status() - return response.json() + return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json()) except httpx.HTTPStatusError as err: raise ValueError(f"HTTP error occurred: {err.response.text}") except httpx.TimeoutException: @@ -533,7 +544,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): try: response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8")) response.raise_for_status() - return response.json() + return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json()) except httpx.HTTPStatusError as err: raise ValueError(f"HTTP error occurred: {err.response.text}") except httpx.TimeoutException: @@ -613,7 +624,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): try: response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8")) response.raise_for_status() - return response.json() + return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json()) except httpx.HTTPStatusError as err: raise ValueError(f"HTTP error occurred: {err.response.text}") except httpx.TimeoutException: diff --git a/litellm/types/integrations/rag/bedrock_knowledgebase.py b/litellm/types/integrations/rag/bedrock_knowledgebase.py index 7156d8101e1..7288eca03b4 100644 --- a/litellm/types/integrations/rag/bedrock_knowledgebase.py +++ b/litellm/types/integrations/rag/bedrock_knowledgebase.py @@ -1,4 +1,4 @@ -from typing import Any, Literal +from typing import Literal from typing_extensions import ReadOnly, TypedDict @@ -40,7 +40,7 @@ class BedrockKBRetrievalResult(TypedDict, total=False): content: BedrockKBContent | None location: BedrockKBLocation | None score: float | None - metadata: dict[str, Any] | None + metadata: dict[str, object] | None class BedrockKBResponse(TypedDict, total=False): @@ -87,21 +87,21 @@ class BedrockKBMetadataConfiguration(TypedDict, total=False): class BedrockKBModelConfiguration(TypedDict, total=False): """Model configuration for reranking.""" - additionalModelRequestFields: dict[str, Any] | None + additionalModelRequestFields: dict[str, object] | None modelArn: str | None class BedrockKBRerankingConfiguration(TypedDict, total=False): """Configuration for reranking in vector search.""" - bedrockRerankingConfiguration: dict[str, Any] | None # This could be further typed if needed + bedrockRerankingConfiguration: dict[str, object] | None # This could be further typed if needed type: str | None class BedrockKBVectorSearchConfiguration(TypedDict, total=False): """Configuration for vector search.""" - filter: dict[str, Any] | None + filter: dict[str, object] | None implicitFilterConfiguration: BedrockKBImplicitFilterConfiguration | None numberOfResults: int | None overrideSearchType: str | None diff --git a/litellm/types/llms/bedrock_invoke_agents.py b/litellm/types/llms/bedrock_invoke_agents.py index 62f3c18ca7a..b334c986d6b 100644 --- a/litellm/types/llms/bedrock_invoke_agents.py +++ b/litellm/types/llms/bedrock_invoke_agents.py @@ -4,7 +4,7 @@ Type definitions for AWS Bedrock Invoke Agent API responses. https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agent-runtime_InvokeAgent.html """ -from typing import Any, Final +from typing import Final from typing_extensions import TypedDict @@ -39,7 +39,7 @@ class InvokeAgentModelInvocationInput(TypedDict, total=False): """Model invocation input details.""" foundationModel: str | None - inferenceConfiguration: dict[str, Any] | None + inferenceConfiguration: dict[str, object] | None text: str | None traceId: str | None type: str | None @@ -49,9 +49,9 @@ class InvokeAgentModelInvocationOutput(TypedDict, total=False): """Model invocation output details.""" metadata: InvokeAgentMetadata | None - parsedResponse: dict[str, Any] | None - rawResponse: dict[str, Any] | None - reasoningContent: dict[str, Any] | None + parsedResponse: dict[str, object] | None + rawResponse: dict[str, object] | None + reasoningContent: dict[str, object] | None traceId: str | None diff --git a/litellm/types/llms/gemini.py b/litellm/types/llms/gemini.py index c300c26bcd3..2ad81e64e05 100644 --- a/litellm/types/llms/gemini.py +++ b/litellm/types/llms/gemini.py @@ -1,5 +1,6 @@ +from collections.abc import Mapping from enum import Enum -from typing import Any, Literal +from typing import Literal from typing_extensions import Required, TypedDict @@ -235,7 +236,7 @@ class GeminiVideoGenerationInstance(TypedDict, total=False): """Instance data for Gemini video generation request""" prompt: Required[str] - image: dict[str, Any] + image: Mapping[str, object] class GeminiVideoGenerationParameters(LiteLLMBaseModel): @@ -263,7 +264,7 @@ class GeminiVideoGenerationParameters(LiteLLMBaseModel): negativePrompt: str | None = None """Text describing what not to include in the video.""" - lastFrame: Any | None = None + lastFrame: object | None = None """ The final image for interpolation video to transition. Should be used with the 'image' parameter. @@ -275,7 +276,7 @@ class GeminiVideoGenerationParameters(LiteLLMBaseModel): Only supported in Veo 3.1 (list of VideoGenerationReferenceImage objects). """ - video: Any | None = None + video: object | None = None """ Video to be used for video extension (Video object). Only supported in Veo 3.1 & Veo 3 Fast. @@ -354,5 +355,5 @@ class GeminiLongRunningOperationResponse(LiteLLMBaseModel): response: GeminiOperationResponse | None = None """Response object when operation is complete""" - error: dict[str, Any] | None = None + error: dict[str, object] | None = None """Error details if operation failed""" diff --git a/litellm/types/llms/langgraph.py b/litellm/types/llms/langgraph.py index 7e34329992d..83f7bb090d1 100644 --- a/litellm/types/llms/langgraph.py +++ b/litellm/types/llms/langgraph.py @@ -4,7 +4,8 @@ Type definitions for LangGraph API. LangGraph provides a streaming and non-streaming API for running agents. """ -from typing import Any, Literal +from collections.abc import Mapping +from typing import Literal from typing_extensions import TypedDict @@ -29,8 +30,8 @@ class LangGraphRequest(TypedDict, total=False): assistant_id: str input: LangGraphInput stream_mode: str | None - config: dict[str, Any] | None - metadata: dict[str, Any] | None + config: Mapping[str, object] | None + metadata: Mapping[str, object] | None # Response Types - Streaming @@ -38,7 +39,7 @@ class LangGraphStreamEvent(TypedDict, total=False): """Single event in a LangGraph stream response.""" event: str - data: Any + data: object # Response Types - Non-streaming @@ -55,7 +56,7 @@ class LangGraphResponse(TypedDict, total=False): """Non-streaming response structure from LangGraph.""" messages: list[LangGraphResponseMessage] - values: dict[str, Any] + values: dict[str, object] # Parsed response for internal use diff --git a/litellm/types/mcp_server/tool_registry.py b/litellm/types/mcp_server/tool_registry.py index 738d37c9503..080e27b6ed3 100644 --- a/litellm/types/mcp_server/tool_registry.py +++ b/litellm/types/mcp_server/tool_registry.py @@ -1,5 +1,5 @@ from collections.abc import Callable -from typing import Any, ClassVar +from typing import ClassVar from pydantic import ConfigDict, Field @@ -10,7 +10,7 @@ class MCPTool(LiteLLMBaseModel): model_config: ClassVar[ConfigDict] = ConfigDict(arbitrary_types_allowed=True) name: str description: str - input_schema: dict[str, Any] + input_schema: dict[str, object] handler: Callable server_id: str | None = Field(default=None, frozen=True) @@ -18,18 +18,18 @@ class MCPTool(LiteLLMBaseModel): class ToolSchema(LiteLLMBaseModel): name: str description: str - inputSchema: dict[str, Any] + inputSchema: dict[str, object] class ListToolsResponse(LiteLLMBaseModel): tools: list[ToolSchema] nextCursor: str | None = None - _meta: dict[str, Any] | None = None + _meta: dict[str, object] | None = None class CallToolRequest(LiteLLMBaseModel): method: str = "tools/call" - params: dict[str, Any] + params: dict[str, object] class ContentItem(LiteLLMBaseModel): diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/enkryptai.py b/litellm/types/proxy/guardrails/guardrail_hooks/enkryptai.py index d1eba1dd4a9..f936c9d85cf 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/enkryptai.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/enkryptai.py @@ -1,5 +1,3 @@ -from typing import Any - from pydantic import Field from typing_extensions import TypedDict @@ -20,7 +18,7 @@ class EnkryptAIPolicyViolationDetail(TypedDict, total=False): class EnkryptAIPIIDetail(TypedDict, total=False): """Details for PII detection.""" - pii: dict[str, Any] + pii: dict[str, object] class EnkryptAIToxicityDetail(TypedDict, total=False): @@ -85,8 +83,8 @@ class EnkryptAIResponseDetails(TypedDict, total=False): toxicity: EnkryptAIToxicityDetail | None keyword_detected: EnkryptAIKeywordDetail | None bias: EnkryptAIBiasDetail | None - prompt_injection: dict[str, Any] | None - jailbreak: dict[str, Any] | None + prompt_injection: dict[str, object] | None + jailbreak: dict[str, object] | None class EnkryptAIResponse(TypedDict, total=False): @@ -107,7 +105,7 @@ class EnkryptAIProcessedResult(TypedDict): | EnkryptAIToxicityDetail | EnkryptAIKeywordDetail | EnkryptAIBiasDetail - | dict[str, Any], + | dict[str, object], ] diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py index 1890c68b30d..c36644ba536 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py @@ -242,6 +242,33 @@ def test_transform_response_hands_json_mode_to_nova(): assert json.loads(result.choices[0].message.content) == {"city": "Paris", "temperature": 21} +@pytest.mark.parametrize(("finish_reason", "expected"), [("MAX_TOKENS", "length"), (None, "stop")]) +def test_transform_response_accepts_null_cohere_finish_reason(finish_reason: str | None, expected: str) -> None: + from litellm.types.utils import ModelResponse + + raw_response: Final = httpx.Response( + 200, + json={"generations": [{"text": "Hello there", "finish_reason": finish_reason}]}, + request=httpx.Request("POST", "https://bedrock"), + ) + + result: Final = AmazonInvokeConfig().transform_response( + model="invoke/cohere.command-text-v14", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + encoding=None, + api_key=None, + ) + + assert result.choices[0].message.content == "Hello there" + assert result.choices[0].finish_reason == expected + + def _stream_invoke_completion_with_spied_client(**kwargs: object) -> tuple[MagicMock, MagicMock]: mock_response = MagicMock() mock_response.status_code = 200 diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_twelvelabs_pegasus_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_twelvelabs_pegasus_transformation.py index d90e9933b88..5b3cb2e0c92 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_twelvelabs_pegasus_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_twelvelabs_pegasus_transformation.py @@ -1,6 +1,13 @@ +from typing import Final +from unittest.mock import Mock + +import httpx +import pytest + from litellm.llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import ( AmazonTwelveLabsPegasusConfig, ) +from litellm.types.utils import ModelResponse def _make_messages() -> list[dict]: @@ -84,3 +91,24 @@ def test_transform_request_includes_s3_media(): s3_location = request["mediaSource"]["s3Location"] assert s3_location["uri"] == "s3://test-bucket/video.mp4" assert s3_location["bucketOwner"] == "123456789012" + + +@pytest.mark.parametrize( + ("finish_reason", "expected"), + [("length", "length"), (None, "stop")], +) +def test_transform_response_accepts_null_finish_reason(finish_reason: str | None, expected: str) -> None: + result: Final = AmazonTwelveLabsPegasusConfig().transform_response( + model="twelvelabs.pegasus-1-2-v1:0", + raw_response=httpx.Response(200, json={"message": "A cat jumps.", "finishReason": finish_reason}), + model_response=ModelResponse(), + logging_obj=Mock(), + request_data={}, + messages=_make_messages(), + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert result.choices[0].message.content == "A cat jumps." + assert result.choices[0].finish_reason == expected From 9e878cfae2edb7a66ecb3d977b2f3320c5955e6d Mon Sep 17 00:00:00 2001 From: moe-berri Date: Wed, 28 Oct 2026 16:26:28 -0700 Subject: [PATCH 22/27] test(gemini): cover every web source for a grounded passage (#48861) * fix(gemini): preserve every grounding web source * test(gemini): cover all grounding sources per passage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Zabir Raihan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_gemini_grounding_annotations_wire.py | 65 +++++++++++++++++++ ...test_vertex_and_google_ai_studio_gemini.py | 33 ++++++++++ 2 files changed, 98 insertions(+) create mode 100644 tests/integration/providers/test_gemini_grounding_annotations_wire.py diff --git a/tests/integration/providers/test_gemini_grounding_annotations_wire.py b/tests/integration/providers/test_gemini_grounding_annotations_wire.py new file mode 100644 index 00000000000..11df7c214f3 --- /dev/null +++ b/tests/integration/providers/test_gemini_grounding_annotations_wire.py @@ -0,0 +1,65 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server + + +def _grounded_reply(request: Request) -> Reply: + body: Final = { + "candidates": [ + { + "content": {"role": "model", "parts": [{"text": "The passage"}]}, + "finishReason": "STOP", + "groundingMetadata": { + "groundingChunks": [ + {"retrievedContext": {"uri": "gs://documents/source"}}, + {"web": {"uri": "https://example.com/first", "title": "First"}}, + {"web": {"uri": "https://example.com/second", "title": "Second"}}, + ], + "groundingSupports": [ + {"segment": {"startIndex": 4, "endIndex": 11}, "groundingChunkIndices": [0, 99, 2, 1]} + ], + }, + } + ], + "usageMetadata": {"promptTokenCount": 5, "candidatesTokenCount": 2, "totalTokenCount": 7}, + } + assert json.loads(request.body)["contents"] == [{"role": "user", "parts": [{"text": "Find sources"}]}] + if ":streamGenerateContent" in request.target: + return Reply(content_type="text/event-stream", chunks=(f"data: {json.dumps(body)}\n\n".encode(),)) + return Reply(body=json.dumps(body).encode()) + + +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +def test_chat_grounding_annotations_include_every_web_source(gateway: Gateway, stream: bool) -> None: + with wire_server(_grounded_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model="gemini/gemini-3.8-flash", api_base=wire.url, api_key="synthetic-key") + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "Find sources"}], "stream": stream}, + ) + assert response.status_code == 200, response.text + payloads: Final = ( + [json.loads(line[6:]) for line in response.text.splitlines() if line.startswith("data: {")] + if stream + else [response.json()] + ) + messages: Final = tuple(payload["choices"][0]["delta" if stream else "message"] for payload in payloads) + annotations: Final = tuple(message["annotations"] for message in messages if message.get("annotations")) + assert annotations == ( + [ + { + "type": "url_citation", + "url_citation": { + "start_index": 4, + "end_index": 11, + "url": f"https://example.com/{name.lower()}", + "title": name, + }, + } + for name in ("Second", "First") + ], + ), response.text diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 1546d1e9fe0..3e1bca18f08 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -3514,6 +3514,39 @@ def test_vertex_ai_annotation_conversion(): assert "Current time" in time_annotation["url_citation"]["title"] +@pytest.mark.parametrize( + "indices, expected_indices", + [([2, 0], [2, 0]), ([1, 99, -1, 2, 0], [2, 0]), ([0], [0]), ([], []), ([1, 99], [])], + ids=["multiple-sources", "non-web-first", "single-source", "empty", "no-web"], +) +def test_grounding_annotations_preserve_all_web_sources(indices: list[int], expected_indices: list[int]) -> None: + chunks: Final = [ + {"web": {"uri": "https://example.com/first", "title": "First"}}, + {"retrievedContext": {"uri": "gs://documents/source"}}, + {"web": {"uri": "https://example.com/second", "title": "Second"}}, + ] + metadata: Final = { + "groundingChunks": chunks, + "groundingSupports": [ + {"segment": {"startIndex": 4, "endIndex": 11}, "groundingChunkIndices": indices} + ], + } + annotations: Final = VertexGeminiConfig._convert_grounding_metadata_to_annotations([metadata], "The passage") + expected: Final = [ + { + "type": "url_citation", + "url_citation": { + "start_index": 4, + "end_index": 11, + "url": chunks[index]["web"]["uri"], + "title": chunks[index]["web"]["title"], + }, + } + for index in expected_indices + ] + assert json.dumps(annotations) == json.dumps(expected) + + def test_vertex_ai_annotation_empty_grounding_metadata(): """ Test handling of empty or missing grounding metadata. From 96fe78f96ecb6d0a5f92a6b7e68a3b770bb25213 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 28 Oct 2026 16:32:45 -0700 Subject: [PATCH 23/27] fix(router): reject string fallback targets at Router init (LIT-10238) (#48943) * fix(router): reject string fallback targets at Router init (LIT-10238) * fix(router): keep direct target fallback entries when rejecting string targets * test(router): set string fallbacks after init in cooldown tolerance tests --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 15 +++++- .../router_utils/test_cooldown_handlers.py | 7 +-- tests/unit/test_router/test_router.py | 48 +++++++++++++++++++ 3 files changed, 63 insertions(+), 7 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 5ca09499c3b..408cb20945f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1370,7 +1370,7 @@ class Router: }, ] - router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": "openai-gpt-3.5-turbo"}]) + router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": ["openai-gpt-3.5-turbo"]}]) ``` """ @@ -1596,6 +1596,10 @@ class Router: if default_fallbacks is not None or litellm.default_fallbacks is not None: _fallbacks = default_fallbacks or litellm.default_fallbacks + if isinstance(_fallbacks, str): + raise TypeError( + f"default_fallbacks must be a list of model names, got '{_fallbacks}'. Use ['{_fallbacks}']" + ) if self.fallbacks is not None: self.fallbacks.append({"*": _fallbacks}) else: @@ -2839,7 +2843,7 @@ class Router: self._initialize_core_endpoints() self._initialize_specialized_endpoints() - def validate_fallbacks(self, fallback_param: list | None): + def validate_fallbacks(self, fallback_param: Sequence[Mapping[str, object]] | None): """ Validate the fallbacks parameter. """ @@ -2852,6 +2856,13 @@ class Router: raise ValueError( f"Dictionary '{fallback_dict}' must have exactly one key, but has {len(fallback_dict)} keys." ) + ((model_group, targets),) = fallback_dict.items() + if isinstance(targets, list) or check_non_standard_fallback_format(fallbacks=[fallback_dict]): + continue + raise TypeError( + f"Fallbacks for '{model_group}' must be a list of model names, got '{targets}'. " + f"Use {{'{model_group}': ['{targets}']}}" + ) def _add_encrypted_content_affinity_check(self, enable_global_affinity: bool) -> None: def _move_before_deployment_affinity( diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 24459a5a250..2e1e3acaabb 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -3150,8 +3150,6 @@ BENCHED: Final = (litellm.Timeout, litellm.Timeout, RouterRateLimitError, Router {}, BENCHED, ), - ({"allowed_fails": 1, "fallbacks": [{"test-model": "backup-model"}]}, {}, BENCHED), - ({"allowed_fails": 1, "default_fallbacks": "backup-model"}, {}, BENCHED), ], ids=[ "allowed_fails", @@ -3160,8 +3158,6 @@ BENCHED: Final = (litellm.Timeout, litellm.Timeout, RouterRateLimitError, Router "policy_names_timeout", "allowed_fails_with_a_fallback_group", "allowed_fails_with_a_dict_fallback_target", - "allowed_fails_with_a_string_fallback_target", - "allowed_fails_with_a_string_default_fallbacks", ], ) @pytest.mark.asyncio @@ -3207,7 +3203,8 @@ async def test_router_allowed_fails_benches_a_lone_deployment_only_when_the_poli def test_has_fallback_group_reads_string_targets_and_skips_unparseable_entries( fallbacks: Sequence[Mapping[str, object]], expected: bool ) -> None: - router: Final = Router(model_list=[TIMED_OUT_DEPLOYMENT], fallbacks=list(fallbacks)) + router: Final = Router(model_list=[TIMED_OUT_DEPLOYMENT]) + router.fallbacks = list(fallbacks) assert _has_fallback_group(router, "test-model") is expected diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 3c7a222d9a1..300c9cff6ff 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -31228,3 +31228,51 @@ async def test_anthropic_caller_rate_limit_does_not_block_other_credentials( finally: await GLOBAL_LOGGING_WORKER.flush() await client.close() + + +_FALLBACK_VALIDATION_MODEL_LIST: Final = ( + {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4", "api_key": "fake-key"}}, + {"model_name": "gpt-3.5", "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "fake-key"}}, +) + + +@pytest.mark.parametrize( + ("router_kwargs", "expected_message"), + [ + ({"fallbacks": [{"gpt-4": "gpt-3.5"}]}, "Fallbacks for 'gpt-4' must be a list of model names, got 'gpt-3.5'"), + ({"default_fallbacks": "gpt-3.5"}, "default_fallbacks must be a list of model names, got 'gpt-3.5'"), + ( + {"content_policy_fallbacks": [{"gpt-4": "gpt-3.5"}]}, + "Fallbacks for 'gpt-4' must be a list of model names, got 'gpt-3.5'", + ), + ], + ids=["string_fallback_target", "string_default_fallbacks", "string_content_policy_fallback_target"], +) +def test_router_rejects_a_string_fallback_target_at_init( + router_kwargs: Mapping[str, object], expected_message: str +) -> None: + with pytest.raises(TypeError, match="must be a list of model names") as excinfo: + Router(model_list=list(_FALLBACK_VALIDATION_MODEL_LIST), **router_kwargs) + + assert expected_message in str(excinfo.value) + + +@pytest.mark.parametrize( + ("router_kwargs", "expected_fallbacks"), + [ + ({"fallbacks": [{"gpt-4": ["gpt-3.5"]}]}, [{"gpt-4": ["gpt-3.5"]}]), + ({"default_fallbacks": ["gpt-3.5"]}, [{"*": ["gpt-3.5"]}]), + ( + {"fallbacks": [{"gpt-4": [{"model": "gpt-3.5", "api_key": "other-key"}]}]}, + [{"gpt-4": [{"model": "gpt-3.5", "api_key": "other-key"}]}], + ), + ({"fallbacks": [{"model": "gpt-3.5"}]}, [{"model": "gpt-3.5"}]), + ], + ids=["list_fallback_target", "list_default_fallbacks", "dict_override_fallback_target", "direct_target_entry"], +) +def test_router_keeps_supported_fallback_shapes( + router_kwargs: Mapping[str, object], expected_fallbacks: list[dict[str, object]] +) -> None: + router: Final = Router(model_list=list(_FALLBACK_VALIDATION_MODEL_LIST), **router_kwargs) + + assert router.fallbacks == expected_fallbacks From ab07b7207e37e52f1ac8126817d101c90f5d01a0 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:59:57 -0700 Subject: [PATCH 24/27] fix(vertex-ai): add batch above 200k and image output batch prices for gemini models (#48945) * fix(vertex-ai): add batch and flex above 200k and image output tier prices for gemini models * fix(vertex-ai): drop flex above 200k and image priority fields the cost map schema rejects Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 28 ++++++++++++++++--- model_prices_and_context_window.json | 28 ++++++++++++++++--- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index dc043dc4d26..8719565a96f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -28853,6 +28853,7 @@ "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -28910,6 +28911,7 @@ "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, + "output_cost_per_image_token_batches": 6e-05, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, @@ -28963,12 +28965,13 @@ "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, + "output_cost_per_image_token_batches": 6e-05, "output_cost_per_token": 1.2e-05, "output_cost_per_token_priority": 2.16e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, - "source": "https://ai.google.dev/gemini-api/docs/pricing", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/images/generations", @@ -29014,6 +29017,7 @@ "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, + "output_cost_per_image_token_batches": 3e-05, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, @@ -29059,9 +29063,10 @@ "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, + "output_cost_per_image_token_batches": 3e-05, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/images/generations", @@ -29106,6 +29111,7 @@ "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, "output_cost_per_token_flex": 7.5e-07, @@ -29672,10 +29678,12 @@ }, "search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"}, "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, "input_cost_per_token_above_200k_tokens_priority": 4.5e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_flex": 6.25e-07, "input_cost_per_token_priority": 2.25e-06, + "output_cost_per_token_above_200k_tokens_batches": 7.5e-06, "output_cost_per_token_above_200k_tokens_priority": 2.7e-05, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_flex": 5e-06, @@ -29690,6 +29698,7 @@ "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, @@ -29701,6 +29710,7 @@ "output_cost_per_image": 0.00012, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, + "output_cost_per_token_above_200k_tokens_batches": 9e-06, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, @@ -30229,6 +30239,7 @@ "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, @@ -30240,6 +30251,7 @@ "output_cost_per_image": 0.00012, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, + "output_cost_per_token_above_200k_tokens_batches": 9e-06, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, @@ -51910,6 +51922,7 @@ "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -51968,6 +51981,7 @@ "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, + "output_cost_per_image_token_batches": 6e-05, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, @@ -52002,13 +52016,14 @@ "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, + "output_cost_per_image_token_batches": 6e-05, "output_cost_per_token": 1.2e-05, "output_cost_per_token_priority": 2.16e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "supports_reasoning": false, - "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/images/generations", @@ -52034,6 +52049,7 @@ "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, + "output_cost_per_image_token_batches": 3e-05, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, @@ -52061,10 +52077,11 @@ "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, + "output_cost_per_image_token_batches": 3e-05, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "supports_reasoning": false, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/images/generations", @@ -52089,6 +52106,7 @@ "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, "output_cost_per_token_flex": 7.5e-07, @@ -74400,6 +74418,7 @@ "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, @@ -74445,6 +74464,7 @@ "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index dc043dc4d26..8719565a96f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -28853,6 +28853,7 @@ "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -28910,6 +28911,7 @@ "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, + "output_cost_per_image_token_batches": 6e-05, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, @@ -28963,12 +28965,13 @@ "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, + "output_cost_per_image_token_batches": 6e-05, "output_cost_per_token": 1.2e-05, "output_cost_per_token_priority": 2.16e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, - "source": "https://ai.google.dev/gemini-api/docs/pricing", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/images/generations", @@ -29014,6 +29017,7 @@ "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, + "output_cost_per_image_token_batches": 3e-05, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, @@ -29059,9 +29063,10 @@ "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, + "output_cost_per_image_token_batches": 3e-05, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/images/generations", @@ -29106,6 +29111,7 @@ "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, "output_cost_per_token_flex": 7.5e-07, @@ -29672,10 +29678,12 @@ }, "search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"}, "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, "input_cost_per_token_above_200k_tokens_priority": 4.5e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_flex": 6.25e-07, "input_cost_per_token_priority": 2.25e-06, + "output_cost_per_token_above_200k_tokens_batches": 7.5e-06, "output_cost_per_token_above_200k_tokens_priority": 2.7e-05, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_flex": 5e-06, @@ -29690,6 +29698,7 @@ "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, @@ -29701,6 +29710,7 @@ "output_cost_per_image": 0.00012, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, + "output_cost_per_token_above_200k_tokens_batches": 9e-06, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, @@ -30229,6 +30239,7 @@ "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, @@ -30240,6 +30251,7 @@ "output_cost_per_image": 0.00012, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, + "output_cost_per_token_above_200k_tokens_batches": 9e-06, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, @@ -51910,6 +51922,7 @@ "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -51968,6 +51981,7 @@ "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, + "output_cost_per_image_token_batches": 6e-05, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, @@ -52002,13 +52016,14 @@ "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, + "output_cost_per_image_token_batches": 6e-05, "output_cost_per_token": 1.2e-05, "output_cost_per_token_priority": 2.16e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "supports_reasoning": false, - "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/images/generations", @@ -52034,6 +52049,7 @@ "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, + "output_cost_per_image_token_batches": 3e-05, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, @@ -52061,10 +52077,11 @@ "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, + "output_cost_per_image_token_batches": 3e-05, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "supports_reasoning": false, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/images/generations", @@ -52089,6 +52106,7 @@ "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, "output_cost_per_token_flex": 7.5e-07, @@ -74400,6 +74418,7 @@ "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, @@ -74445,6 +74464,7 @@ "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, From d06b889d413237881e3dc27486e4345dd7aa18f9 Mon Sep 17 00:00:00 2001 From: rj-cairn <349020452+rj-cairn@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:00:29 -0700 Subject: [PATCH 25/27] fix(streaming): keep merged think tags balanced and split SSE lines only on CR/LF (#48755) * fix(vertex_ai): split Gemini SSE lines on CR/LF only so U+2028 in text keeps the stream The Gemini and Vertex AI streaming path read the SSE body with httpx iter_lines()/aiter_lines(). Those follow str.splitlines(), which also breaks on U+0085, U+2028, U+2029, \x0b, \x0c and \x1c-\x1e. JSON leaves U+0085, U+2028 and U+2029 unescaped inside strings, so a data: line whose text holds one of them was cut in two. The tail is not a data: line and the head never parses, and the accumulate-and-retry fallback then carries the unparsed prefix forward, so every later event is appended to it and the stream ends with no more content and no usage. SSE lines end only at CRLF, LF or CR. The new iter_sse_lines/aiter_sse_lines split decoded text on exactly those, treating a CR at the end of one chunk and an LF at the start of the next as one break. The first request and every long-decode hop now read through them. Co-Authored-By: Claude Opus 5.5 * fix(streaming): keep merged think tags balanced and split shared SSE lines only on CR/LF Close an open block on the first content or tool-call delta and at stream end, reopen it when reasoning returns, and route the shared chat completions stream through the SSE line splitter so U+2028 and U+2029 inside JSON no longer drop chunks. * fix(streaming): exact-match the SSE [DONE] marker in BaseModelResponseIterator * fix(streaming): split every SSE reader on CR/LF only and keep the logged think block closed Switch the remaining httpx stream readers (Anthropic chat, Google GenAI, Interactions, Codestral, openai_like, Predibase, LangGraph, Vertex Agent Engine, A2A providers, Azure AI agents, sandboxes, the A2A endpoint, and others) from iter_lines/aiter_lines to iter_sse_lines/aiter_sse_lines, so a raw U+2028 or U+2029 inside a JSON string no longer splits an event. Keep the chunk that closes a think block in the logged and cached response when a stream ends without a finish_reason, close the block before a legacy function_call, and make sync Codestral FIM streaming send stream=true and use the text-completion-codestral stream parser. Test fakes that only mocked iter_lines now stream real httpx bytes and assert the content they feed in. * fix(streaming): split buffered SSE parsers only on CR and LF, shard the codestral tests Buffered SSE and JSONL parsers used str.splitlines(), which also breaks on U+2028, U+2029 and U+0085 that JSON allows raw inside strings. They now use split_sse_lines from sse_lines.py, which also gains a str.split fast path. Adds tests/unit/llms/codestral to the unit workflow shard. * fix(streaming): keep U+2028 in restamped Anthropic frames and fold CRLF before splitting SSE lines * test: type the new stream tests, drop a comment, and benchmark the SSE line splitter * test: type the iterator class parameters in the line-separator stream tests * fix(responses): stream tool call arguments opened in a text chunk * fix(responses): hold a tool call that shares a chunk with text until the text ends * test: type the a2a SSE handler helper and the sandbox fake responses --------- Co-authored-by: Tim Vink Co-authored-by: Claude Opus 5.5 Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: michelligabriele --- .github/workflows/test-unit.yml | 1 + .../bedrock_agentcore/transformation.py | 5 +- .../providers/watsonx_orchestrate/handler.py | 5 +- .../transformation.py | 3 +- litellm/google_genai/streaming_iterator.py | 9 +- litellm/interactions/streaming_iterator.py | 5 +- .../litellm_core_utils/streaming_handler.py | 57 +++--- litellm/llms/anthropic/chat/handler.py | 5 +- .../anthropic/messages/stream_assembly.py | 3 +- litellm/llms/azure_ai/agents/handler.py | 3 +- litellm/llms/base_llm/base_model_iterator.py | 2 +- .../base_llm/guardrail_translation/utils.py | 3 +- .../llms/base_llm/sandbox/transformation.py | 3 +- litellm/llms/bedrock/files/transformation.py | 3 +- .../guardrail_translation/handler.py | 3 +- litellm/llms/bedrock/search/transformation.py | 5 +- .../llms/chatgpt/responses/transformation.py | 3 +- litellm/llms/codestral/completion/handler.py | 9 +- litellm/llms/custom_httpx/aiohttp_handler.py | 3 +- litellm/llms/custom_httpx/llm_http_handler.py | 5 +- litellm/llms/custom_httpx/sse_lines.py | 74 ++++++++ .../guardrail_translation/handler.py | 3 +- .../huggingface/embedding/transformation.py | 3 +- litellm/llms/langgraph/chat/sse_iterator.py | 5 +- litellm/llms/oobabooga/chat/oobabooga.py | 3 +- litellm/llms/openai_like/chat/handler.py | 9 +- litellm/llms/predibase/chat/handler.py | 5 +- .../vertex_ai/agent_engine/transformation.py | 5 +- .../vertex_and_google_ai_studio_gemini.py | 9 +- .../proxy/agent_endpoints/a2a_endpoints.py | 3 +- .../streaming_model_restamp.py | 4 +- .../anthropic_passthrough_logging_handler.py | 7 +- .../tinyfish_passthrough_logging_handler.py | 3 +- .../managed_id_rewriter.py | 3 +- .../response_polling/background_streaming.py | 3 +- .../streaming_iterator.py | 13 +- tests/benchmarks/test_sse_lines_benchmarks.py | 27 +++ .../test_bedrock_agentcore_a2a.py | 32 ++++ ...test_watsonx_orchestrate_transformation.py | 20 ++- ...responses_transformation_transformation.py | 18 ++ .../test_google_genai_streaming_iterator.py | 83 +++++---- .../test_interactions_streaming_iterator.py | 45 ++++- .../test_streaming_handler.py | 114 +++++++++++- .../chat/test_anthropic_chat_handler.py | 59 ++++++- .../messages/test_stream_assembly.py | 24 +++ .../unit/llms/azure_ai/agents/test_handler.py | 23 ++- .../azure_ai/agents/test_transformation.py | 20 +-- .../guardrail_translation/test_utils.py | 17 ++ .../test_bedrock_files_transformation.py | 14 ++ .../guardrail_translation/test_handler.py | 14 +- .../test_agentcore_search_transformation.py | 15 ++ .../test_chatgpt_responses_transformation.py | 31 ++++ .../test_cloudflare_transformation.py | 47 ++++- tests/unit/llms/codestral/__init__.py | 0 .../llms/codestral/completion/__init__.py | 0 .../test_codestral_completion_handler.py | 50 ++++++ .../llms/custom_httpx/test_aiohttp_handler.py | 37 ++++ .../unit/llms/custom_httpx/test_sse_lines.py | 64 +++++++ .../test_databricks_chat_transformation.py | 14 +- .../test_fireworks_ai_chat_transformation.py | 6 +- .../test_google_genai_guardrail_handler.py | 16 ++ .../chat/test_langgraph_sse_iterator.py | 32 ++++ .../llms/oobabooga/chat/test_oobabooga.py | 29 ++++ .../chat/test_openai_like_chat_handler.py | 37 +++- .../test_snowflake_chat_transformation.py | 30 +--- .../llms/test_predibase_transformation.py | 47 +++++ tests/unit/llms/triton/test_triton.py | 11 +- .../agent_engine/test_transformation.py | 47 ++++- ...test_vertex_and_google_ai_studio_gemini.py | 64 ++++++- .../llms/vertex_ai/test_http_status_201.py | 8 +- .../agent_endpoints/test_a2a_endpoints.py | 138 +++++++-------- .../test_streaming_model_restamp.py | 23 +++ ...t_anthropic_passthrough_logging_handler.py | 48 +++++ ...st_tinyfish_passthrough_logging_handler.py | 7 + .../test_managed_id_rewriter.py | 33 +++- .../test_response_polling_handler.py | 31 +++- .../test_streaming_iterator_transformation.py | 100 +++++++++++ tests/unit/sandbox/test_e2b_sandbox.py | 28 ++- .../unit/sandbox/test_opensandbox_sandbox.py | 13 +- tests/unit/test_groq_streaming_encoding.py | 164 ++++-------------- 80 files changed, 1554 insertions(+), 413 deletions(-) create mode 100644 litellm/llms/custom_httpx/sse_lines.py create mode 100644 tests/benchmarks/test_sse_lines_benchmarks.py create mode 100644 tests/unit/llms/codestral/__init__.py create mode 100644 tests/unit/llms/codestral/completion/__init__.py create mode 100644 tests/unit/llms/codestral/completion/test_codestral_completion_handler.py create mode 100644 tests/unit/llms/custom_httpx/test_sse_lines.py create mode 100644 tests/unit/llms/langgraph/chat/test_langgraph_sse_iterator.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 2ddf21bca17..80d5018fe9a 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -245,6 +245,7 @@ jobs: tests/unit/llms/clarifai tests/unit/llms/claude_code tests/unit/llms/cloudflare + tests/unit/llms/codestral tests/unit/llms/codex tests/unit/llms/cohere tests/unit/llms/cometapi diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py index 1cea0194715..d4400fbb4cc 100644 --- a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py @@ -19,6 +19,7 @@ from litellm.a2a_protocol.utils import ( ) from litellm.exceptions import BadRequestError from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig +from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines RUNTIME_SESSION_ID_MIN_LENGTH: Final = 33 RUNTIME_SESSION_ID_MAX_LENGTH: Final = 256 @@ -50,7 +51,7 @@ _RESERVED_PREFIX_HEADERS: Final[tuple[str, ...]] = ( class _SSELineSource(Protocol): """Minimal streaming-response surface used to read SSE lines.""" - def aiter_lines(self) -> AsyncIterator[str]: ... + def aiter_text(self) -> AsyncIterator[str]: ... def _filter_reserved_headers( @@ -244,7 +245,7 @@ class BedrockAgentCoreA2ATransformation: Yields: Parsed JSON dicts from SSE data lines """ - async for line in response.aiter_lines(): + async for line in aiter_sse_lines(response.aiter_text()): line = line.strip() if not line: continue diff --git a/litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py b/litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py index 0a2871d25cc..3937c47ea3c 100644 --- a/litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py +++ b/litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py @@ -20,6 +20,7 @@ from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, get_async_httpx_client, ) +from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines from litellm.types.llms.custom_http import httpxSpecialProvider _IBM_CLOUD_IAM_URL: Final = "https://iam.cloud.ibm.com/identity/token" @@ -74,7 +75,7 @@ class _WXORun(TypedDict, total=False): class _SSELineSource(Protocol): - def aiter_lines(self) -> AsyncIterator[str]: ... + def aiter_text(self) -> AsyncIterator[str]: ... class _WXOView(TypedDict, total=False): @@ -231,7 +232,7 @@ class WatsonxOrchestrateHandler: async def _accumulate_wxo_sse_text(response: _SSELineSource) -> str: source: Final[_WXOView] = {"sse_source": response} accumulated_text = "" - async for line in source["sse_source"].aiter_lines(): + async for line in aiter_sse_lines(source["sse_source"].aiter_text()): if not line.startswith("data:"): continue data_str = line[5:].strip() diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 47a3a903d82..ee3639ac387 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -34,6 +34,7 @@ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.bridges.completion_transformation import ( CompletionTransformationBridge, ) +from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.responses.litellm_completion_transformation.tool_search import is_anthropic_tool_search_chat_tool from litellm.responses.sse_output_recovery import ( parse_sse_json_chunk, @@ -1351,7 +1352,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): recovered_output_items: Final[dict[int, dict[str, object]]] = {} recovered_text_only_items: Final[dict[int, dict[str, object]]] = {} - for chunk in raw_sse.splitlines(): + for chunk in split_sse_lines(raw_sse): parsed_chunk = parse_sse_json_chunk(chunk) if parsed_chunk is None: continue diff --git a/litellm/google_genai/streaming_iterator.py b/litellm/google_genai/streaming_iterator.py index a22c47cb1e7..08ac7d4f9dc 100644 --- a/litellm/google_genai/streaming_iterator.py +++ b/litellm/google_genai/streaming_iterator.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) @@ -131,9 +132,9 @@ class GoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateContent self.model = model self.generate_content_provider_config = generate_content_provider_config self.litellm_metadata = litellm_metadata - # Gemini streamGenerateContent uses SSE line framing; iter_lines keeps + # Gemini streamGenerateContent uses SSE line framing; iter_sse_lines keeps # large inlineData payloads (e.g. image/jpeg) intact within one event. - self.stream_iterator = response.iter_lines() + self.stream_iterator = iter_sse_lines(response.iter_text()) def __iter__(self): return self @@ -182,9 +183,9 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateCo self.model = model self.generate_content_provider_config = generate_content_provider_config self.litellm_metadata = litellm_metadata - # Gemini streamGenerateContent uses SSE line framing; aiter_lines keeps + # Gemini streamGenerateContent uses SSE line framing; aiter_sse_lines keeps # large inlineData payloads (e.g. image/jpeg) intact within one event. - self.stream_iterator = response.aiter_lines() + self.stream_iterator = aiter_sse_lines(response.aiter_text()) def __aiter__(self): return self diff --git a/litellm/interactions/streaming_iterator.py b/litellm/interactions/streaming_iterator.py index f4ad57bf7b4..c7faa3fc510 100644 --- a/litellm/interactions/streaming_iterator.py +++ b/litellm/interactions/streaming_iterator.py @@ -21,6 +21,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig +from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines from litellm.types.interactions import ( InteractionsAPIStreamingResponse, ) @@ -145,7 +146,7 @@ class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator): litellm_metadata=litellm_metadata, custom_llm_provider=custom_llm_provider, ) - self.stream_iterator = response.aiter_lines() + self.stream_iterator = aiter_sse_lines(response.aiter_text()) def __aiter__(self): return self @@ -212,7 +213,7 @@ class SyncInteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator) litellm_metadata=litellm_metadata, custom_llm_provider=custom_llm_provider, ) - self.stream_iterator = response.iter_lines() + self.stream_iterator = iter_sse_lines(response.iter_text()) def __iter__(self): return self diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 3fb25de43cc..2348d8e57d6 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1263,7 +1263,7 @@ class CustomStreamWrapper: ) if _is_delta_empty: - model_response.choices[0].delta = Delta(content=None) # ensure empty delta chunk returned + model_response.choices[0].delta = Delta(content=self._close_open_think_block()) # get any function call arguments model_response.choices[0].finish_reason = map_finish_reason( finish_reason=self.received_finish_reason @@ -1292,21 +1292,26 @@ class CustomStreamWrapper: if self.merge_reasoning_content_in_choices is not True: return delta: Final = model_response.choices[0].delta - reasoning_content: Final = getattr(delta, "reasoning_content", None) + reasoning_content: Final = getattr(delta, "reasoning_content", None) or "" content: Final = delta.content or "" - if reasoning_content: - self._emitted_disqualifying_content = True - closes_thinking: Final = bool(content) and not self.sent_last_thinking_block - opening: Final = "" if self.sent_first_thinking_block else "" - delta.content = opening + reasoning_content + ("" + content if closes_thinking else "") - self.sent_first_thinking_block = True - self.sent_last_thinking_block = self.sent_last_thinking_block or closes_thinking - elif self.sent_first_thinking_block and not self.sent_last_thinking_block and content: - delta.content = "" + content - self.sent_last_thinking_block = True - + ends_thinking: Final = bool(content or delta.tool_calls or delta.function_call) + inside_think_block: Final = self.sent_first_thinking_block and not self.sent_last_thinking_block + opening: Final = "" if reasoning_content and not inside_think_block else "" + closing: Final = "" if ends_thinking and (reasoning_content or inside_think_block) else "" if hasattr(delta, "reasoning_content"): del delta.reasoning_content + if not reasoning_content and not closing: + return + delta.content = opening + reasoning_content + closing + content + self.sent_first_thinking_block = True + self.sent_last_thinking_block = bool(closing) + self._emitted_disqualifying_content = self._emitted_disqualifying_content or bool(reasoning_content) + + def _close_open_think_block(self) -> str | None: + if not self.sent_first_thinking_block or self.sent_last_thinking_block: + return None + self.sent_last_thinking_block = True + return "" def _dispatch_provider_chunk( self, @@ -1932,8 +1937,20 @@ class CustomStreamWrapper: model_response.choices[0].finish_reason == "stop" and self.tool_call ): # don't overwrite for other - potential error finish reasons model_response.choices[0].finish_reason = "tool_calls" + close_tag: Final = self._close_open_think_block() + if close_tag is not None: + model_response.choices[0].delta.content = close_tag return model_response + def _final_chunk_for_logging(self, processed_chunk: "ModelResponseStream") -> "ModelResponseStream | None": + if self.received_finish_reason is not None or self.intermittent_finish_reason is not None: + return processed_chunk + if processed_chunk.choices[0].delta.content is None: + return None + return processed_chunk.model_copy( + update={"choices": [processed_chunk.choices[0].model_copy(update={"finish_reason": None})]} + ) + def _record_usage_only_chunk(self, model_response: "ModelResponseStream") -> None: """ Keep provider usage-only chunks (e.g. OpenRouter's post-finish chunk, which carries a @@ -2173,10 +2190,9 @@ class CustomStreamWrapper: else: self.sent_last_chunk = True processed_chunk: Final = self.finish_reason_handler() - # The logged response is built from self.chunks; keep a finish_reason the provider sent on its - # last content chunk (stripped there), but never add the synthetic "stop" used when it sent none. - if self.received_finish_reason is not None or self.intermittent_finish_reason is not None: - self.chunks.append(processed_chunk) + final_chunk_for_logging: Final = self._final_chunk_for_logging(processed_chunk) + if final_chunk_for_logging is not None: + self.chunks.append(final_chunk_for_logging) if self.stream_options is None: self._set_final_chunk_usage(processed_chunk) ## LOGGING @@ -2406,10 +2422,9 @@ class CustomStreamWrapper: else: self.sent_last_chunk = True processed_chunk: Final = self.finish_reason_handler() - # The logged response is built from self.chunks; keep a finish_reason the provider sent on its - # last content chunk (stripped there), but never add the synthetic "stop" used when it sent none. - if self.received_finish_reason is not None or self.intermittent_finish_reason is not None: - self.chunks.append(processed_chunk) + final_chunk_for_logging: Final = self._final_chunk_for_logging(processed_chunk) + if final_chunk_for_logging is not None: + self.chunks.append(final_chunk_for_logging) if self.stream_options is None: self._set_final_chunk_usage(processed_chunk) # see sync __next__'s sibling branch: deliberately do NOT restore diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 13ceea3e31b..8a82ed98991 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -38,6 +38,7 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, get_httpx_client, ) +from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines from litellm.types.llms.anthropic import ( ContentBlockDelta, ContentBlockStart, @@ -233,7 +234,7 @@ async def make_call( raise AnthropicError(status_code=500, message=str(e)) completion_stream: Final = ModelResponseIterator( - streaming_response=response.aiter_lines(), + streaming_response=aiter_sse_lines(response.aiter_text()), sync_stream=False, json_mode=json_mode, speed=speed, @@ -304,7 +305,7 @@ def make_sync_call( ) completion_stream: Final = ModelResponseIterator( - streaming_response=response.iter_lines(), + streaming_response=iter_sse_lines(response.iter_text()), sync_stream=True, json_mode=json_mode, speed=speed, diff --git a/litellm/llms/anthropic/messages/stream_assembly.py b/litellm/llms/anthropic/messages/stream_assembly.py index 90bdad8f4bc..c103eb0f975 100644 --- a/litellm/llms/anthropic/messages/stream_assembly.py +++ b/litellm/llms/anthropic/messages/stream_assembly.py @@ -13,6 +13,7 @@ from litellm.llms.anthropic.chat.handler import ( ) from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.messages.native.utils import MESSAGE_DELTA_OPTIONAL_KEYS +from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.types.utils import ( Choices, Message, @@ -269,7 +270,7 @@ def build_complete_streaming_response_legacy( def extract_sse_data(event_str: str) -> dict | None: """Parse the JSON object from the ``data:`` line of an Anthropic SSE event.""" - for line in event_str.splitlines(): + for line in split_sse_lines(event_str): stripped = line.strip() if stripped.startswith("data:"): payload = stripped[len("data:") :].strip() diff --git a/litellm/llms/azure_ai/agents/handler.py b/litellm/llms/azure_ai/agents/handler.py index 94f1399f01b..423a09c5c92 100644 --- a/litellm/llms/azure_ai/agents/handler.py +++ b/litellm/llms/azure_ai/agents/handler.py @@ -35,6 +35,7 @@ from litellm.llms.azure_ai.agents.transformation import ( AzureAIAgentsConfig, AzureAIAgentsError, ) +from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines from litellm.types.llms.openai import ( ChatCompletionAnnotation, ChatCompletionAnnotationURLCitation, @@ -632,7 +633,7 @@ class AzureAIAgentsHandler: current_event = None - async for line in response.aiter_lines(): + async for line in aiter_sse_lines(response.aiter_text()): line = line.strip() if line.startswith("event:"): diff --git a/litellm/llms/base_llm/base_model_iterator.py b/litellm/llms/base_llm/base_model_iterator.py index 032951be74a..3afc685b3dd 100644 --- a/litellm/llms/base_llm/base_model_iterator.py +++ b/litellm/llms/base_llm/base_model_iterator.py @@ -96,7 +96,7 @@ class BaseModelResponseIterator: released (and a backend like vLLM aborts generation) when the stream is abandoned before its natural end. - ``streaming_response`` is usually a bare ``aiter_lines()`` generator + ``streaming_response`` is usually a bare ``aiter_sse_lines()`` generator that holds no reference to the response, so the handler that owns the response attaches it here after construction.""" if self.http_response is not None: diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index 725146fe894..fb886e42780 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -6,6 +6,7 @@ from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row ha from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage @@ -24,7 +25,7 @@ def _anthropic_stream_chunk_events(item: object) -> list[dict]: events: Final[list[dict]] = [] for block in chunk.split("\n\n"): - for line in block.splitlines(): + for line in split_sse_lines(block): stripped = line.strip() if not stripped.startswith("data:"): continue diff --git a/litellm/llms/base_llm/sandbox/transformation.py b/litellm/llms/base_llm/sandbox/transformation.py index fe0e3abb87e..9b9b4e0a947 100644 --- a/litellm/llms/base_llm/sandbox/transformation.py +++ b/litellm/llms/base_llm/sandbox/transformation.py @@ -12,6 +12,7 @@ from typing import Final import httpx from pydantic import Field, PrivateAttr +from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines from litellm.types.llms.base import LiteLLMPydanticObjectBase SANDBOX_MAX_OUTPUT_BYTES: Final = 10 * 1024 * 1024 @@ -99,7 +100,7 @@ class BaseSandboxConfig: async def _read_capped_lines(self, response: httpx.Response) -> list[str]: lines: Final[list[str]] = [] total = 0 - async for line in response.aiter_lines(): + async for line in aiter_sse_lines(response.aiter_text()): total += len(line.encode("utf-8")) if total > SANDBOX_MAX_OUTPUT_BYTES: raise ValueError( diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 8c4702bcf99..0624c2c65db 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -49,6 +49,7 @@ from litellm.llms.base_llm.files.transformation import ( BaseFilesConfig, LiteLLMLoggingObj, ) +from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.bedrock import AwsAuthParams, BedrockBatchRecordKind from litellm.types.llms.openai import ( @@ -400,7 +401,7 @@ def _batch_input_line(line_number: int, line: str) -> _BatchInputLine: def _non_blank_lines(file_content: str) -> Iterator[tuple[int, str]]: - numbered_lines: Final = enumerate(file_content.splitlines(), start=1) + numbered_lines: Final = enumerate(split_sse_lines(file_content), start=1) return ((line_number, line) for line_number, line in numbered_lines if line.strip()) diff --git a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py index 1b01b0ec6d0..3a65a8928bf 100644 --- a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py +++ b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py @@ -45,6 +45,7 @@ from litellm.llms.bedrock.passthrough.guardrail_translation.structured_edits imp structured_edits, tool_edits, ) +from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.proxy.guardrails.auto_router_compression import is_compression_guardrail from litellm.types.utils import GenericGuardrailAPIInputs @@ -947,7 +948,7 @@ def _invoke_frames(items: Sequence[object]) -> tuple[InvokeStreamFrame, ...]: def _sse_events(sse: bytes) -> tuple[Mapping[str, object], ...]: return tuple( _JSON_OBJECT.validate_python(json.loads(line[len("data:") :])) - for line in sse.decode().splitlines() + for line in split_sse_lines(sse.decode()) if line.startswith("data:") ) diff --git a/litellm/llms/bedrock/search/transformation.py b/litellm/llms/bedrock/search/transformation.py index 565fc12aa1e..6f72cadad67 100644 --- a/litellm/llms/bedrock/search/transformation.py +++ b/litellm/llms/bedrock/search/transformation.py @@ -47,6 +47,7 @@ from litellm.llms.base_llm.search.transformation import ( ) from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.secret_managers.main import get_secret_str # AgentCore web-search rejects queries longer than 200 characters @@ -146,7 +147,9 @@ def _iter_sse_events(text: str) -> Iterator[Mapping[str, object]]: progress notifications before the JSON-RPC response. """ for chunk in _SSE_EVENT_SEPARATOR.split(text): - payload = "\n".join(line[len("data:") :].lstrip() for line in chunk.splitlines() if line.startswith("data:")) + payload = "\n".join( + line[len("data:") :].lstrip() for line in split_sse_lines(chunk) if line.startswith("data:") + ) if not payload: continue try: diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 040b7b8c1d8..72b5cde21d1 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -11,6 +11,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo safe_convert_created_field, ) from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.llms.openai.common_utils import OpenAIError from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.responses.sse_output_recovery import ( @@ -199,7 +200,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): error_message = None streamed_output_items: Final[dict[int, dict[str, object]]] = {} text_only_output_items: Final[dict[int, dict[str, object]]] = {} - for chunk in body_text.splitlines(): + for chunk in split_sse_lines(body_text): parsed_chunk = parse_sse_json_chunk(chunk) if parsed_chunk is None: continue diff --git a/litellm/llms/codestral/completion/handler.py b/litellm/llms/codestral/completion/handler.py index 6bebfe2d444..6b172e77295 100644 --- a/litellm/llms/codestral/completion/handler.py +++ b/litellm/llms/codestral/completion/handler.py @@ -20,6 +20,7 @@ from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, get_async_httpx_client, ) +from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines from litellm.types.utils import TextChoices from litellm.utils import CustomStreamWrapper, TextCompletionResponse @@ -109,7 +110,7 @@ async def make_call( if response.status_code != 200: raise TextCompletionCodestralError(status_code=response.status_code, message=response.text) - completion_stream: Final = response.aiter_lines() + completion_stream: Final = aiter_sse_lines(response.aiter_text()) # LOGGING logging_obj.post_call( input=messages, @@ -339,13 +340,13 @@ class CodestralTextCompletion: response = litellm.module_level_client.post( completion_url, headers=headers, - data=json.dumps(data), + data=json.dumps({**data, "stream": True}), stream=stream, ) _response: Final = CustomStreamWrapper( - response.iter_lines(), + iter_sse_lines(response.iter_text()), model, - custom_llm_provider="codestral", + custom_llm_provider="text-completion-codestral", logging_obj=logging_obj, ) return _response diff --git a/litellm/llms/custom_httpx/aiohttp_handler.py b/litellm/llms/custom_httpx/aiohttp_handler.py index 447d7b685f7..4060d84f7f4 100644 --- a/litellm/llms/custom_httpx/aiohttp_handler.py +++ b/litellm/llms/custom_httpx/aiohttp_handler.py @@ -21,6 +21,7 @@ from litellm.llms.custom_httpx.http_handler import ( get_httpx_client, get_ssl_configuration, ) +from litellm.llms.custom_httpx.sse_lines import iter_sse_lines from litellm.types.llms.openai import FileTypes from litellm.types.utils import HttpHandlerRequestFields, ImageResponse, LlmProviders from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager @@ -466,7 +467,7 @@ class BaseLLMAIOHTTPHandler: ) else: completion_stream = provider_config.get_model_response_iterator( - streaming_response=response.iter_lines(), sync_stream=True + streaming_response=iter_sse_lines(response.iter_text()), sync_stream=True ) # LOGGING diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index afa12b58b29..e7da6586b97 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -118,6 +118,7 @@ from litellm.llms.custom_httpx.http_handler import ( get_httpx_client, header_value, ) +from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines from litellm.responses.streaming_iterator import ( BaseResponsesAPIStreamingIterator, MockResponsesAPIStreamingIterator, @@ -1086,7 +1087,7 @@ class BaseLLMHTTPHandler: completion_stream: object = MockResponseIterator(model_response=model_response, json_mode=json_mode) else: completion_stream = provider_config.get_model_response_iterator_for_request( - streaming_response=response.iter_lines(), + streaming_response=iter_sse_lines(response.iter_text()), sync_stream=True, json_mode=json_mode, optional_params=_OPTIONAL_PARAMS.validate_python(optional_params), @@ -1229,7 +1230,7 @@ class BaseLLMHTTPHandler: completion_stream: object = MockResponseIterator(model_response=model_response, json_mode=json_mode) else: completion_stream = provider_config.get_model_response_iterator_for_request( - streaming_response=response.aiter_lines(), + streaming_response=aiter_sse_lines(response.aiter_text()), sync_stream=False, json_mode=json_mode, optional_params=_OPTIONAL_PARAMS.validate_python(optional_params), diff --git a/litellm/llms/custom_httpx/sse_lines.py b/litellm/llms/custom_httpx/sse_lines.py new file mode 100644 index 00000000000..71ea7d621eb --- /dev/null +++ b/litellm/llms/custom_httpx/sse_lines.py @@ -0,0 +1,74 @@ +""" +Split a decoded Server-Sent Events body into lines the way the SSE spec does. + +httpx's `iter_lines()` / `aiter_lines()` follow `str.splitlines()`, which also breaks on U+0085, U+2028, +U+2029, \\x0b, \\x0c and \\x1c-\\x1e. JSON leaves U+0085, U+2028 and U+2029 unescaped inside strings, so a +`data:` line carrying one of them gets cut in half. SSE lines end only at CRLF, LF or CR +(https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream) +""" + +import io +import re +from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator, Sequence +from typing import Final + +_SSE_LINE_BREAK: Final = re.compile(r"\r\n|\r|\n") +_SSE_LINE_WITH_BREAK: Final = re.compile(r"[^\r\n]*(?:\r\n|\r|\n)|[^\r\n]+") + + +def _split_lines(text: str) -> Sequence[str]: + if "\r" not in text: + return text.split("\n") + crlf_folded: Final = text.replace("\r\n", "\n") + if "\r" not in crlf_folded: + return crlf_folded.split("\n") + return _SSE_LINE_BREAK.split(text) + + +class _SSELineSplitter: + def __init__(self) -> None: + self._partial_line: io.StringIO = io.StringIO() + self._after_cr: bool = False + + def feed(self, text: str) -> tuple[str, ...]: + if not text: + return () + fresh: Final = text[1:] if self._after_cr and text.startswith("\n") else text + self._after_cr = fresh.endswith("\r") + *ended, tail = _split_lines(fresh) + if not ended: + _ = self._partial_line.write(tail) + return () + first: Final = self._partial_line.getvalue() + ended[0] + self._partial_line = io.StringIO() + _ = self._partial_line.write(tail) + return (first, *ended[1:]) + + def flush(self) -> tuple[str, ...]: + rest: Final = self._partial_line.getvalue() + self._partial_line = io.StringIO() + return (rest,) if rest else () + + +def iter_sse_lines(text_chunks: Iterable[str]) -> Iterator[str]: + splitter: Final = _SSELineSplitter() + for text in text_chunks: + yield from splitter.feed(text) + yield from splitter.flush() + + +def split_sse_lines(text: str) -> tuple[str, ...]: + return tuple(iter_sse_lines((text,))) + + +def split_sse_lines_keepends(text: str) -> tuple[str, ...]: + return tuple(_SSE_LINE_WITH_BREAK.findall(text)) + + +async def aiter_sse_lines(text_chunks: AsyncIterable[str]) -> AsyncIterator[str]: + splitter: Final = _SSELineSplitter() + async for text in text_chunks: + for line in splitter.feed(text): + yield line + for line in splitter.flush(): + yield line diff --git a/litellm/llms/gemini/google_genai/guardrail_translation/handler.py b/litellm/llms/gemini/google_genai/guardrail_translation/handler.py index 34007bc8fd4..b6b2aa6c28f 100644 --- a/litellm/llms/gemini/google_genai/guardrail_translation/handler.py +++ b/litellm/llms/gemini/google_genai/guardrail_translation/handler.py @@ -23,6 +23,7 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, StreamTransformSink, ) +from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: @@ -121,7 +122,7 @@ def _parse_json_dict_or_none(payload: str) -> Mapping[str, object] | None: def _sse_payload_texts(sse_text: str) -> tuple[str, ...]: return tuple( text - for line in sse_text.splitlines() + for line in split_sse_lines(sse_text) if line.startswith("data:") for payload in (line[len("data:") :].strip(),) if payload and payload != "[DONE]" diff --git a/litellm/llms/huggingface/embedding/transformation.py b/litellm/llms/huggingface/embedding/transformation.py index 1b7d6483739..6e6a117227e 100644 --- a/litellm/llms/huggingface/embedding/transformation.py +++ b/litellm/llms/huggingface/embedding/transformation.py @@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( ) from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.custom_httpx.sse_lines import iter_sse_lines from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage @@ -357,7 +358,7 @@ class HuggingFaceEmbeddingConfig(BaseConfig): api_key: str | None = None, ) -> list[dict[str, str]]: streamed_response: Final = CustomStreamWrapper( - completion_stream=response.iter_lines(), + completion_stream=iter_sse_lines(response.iter_text()), model=model, custom_llm_provider="huggingface", logging_obj=logging_obj, diff --git a/litellm/llms/langgraph/chat/sse_iterator.py b/litellm/llms/langgraph/chat/sse_iterator.py index 575820bceee..6fe355ad98c 100644 --- a/litellm/llms/langgraph/chat/sse_iterator.py +++ b/litellm/llms/langgraph/chat/sse_iterator.py @@ -12,6 +12,7 @@ import httpx from pydantic import TypeAdapter from litellm._logging import verbose_logger +from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices _DECODED_JSON: Final = TypeAdapter(object) @@ -36,12 +37,12 @@ class LangGraphSSEStreamIterator: def __iter__(self): """Initialize sync iteration.""" - self.line_iterator = self.response.iter_lines() + self.line_iterator = iter_sse_lines(self.response.iter_text()) return self def __aiter__(self): """Initialize async iteration.""" - self.async_line_iterator = self.response.aiter_lines() + self.async_line_iterator = aiter_sse_lines(self.response.aiter_text()) return self def _parse_sse_line(self, line: str) -> ModelResponseStream | None: diff --git a/litellm/llms/oobabooga/chat/oobabooga.py b/litellm/llms/oobabooga/chat/oobabooga.py index 8879c20c5eb..b4dc57ff5ed 100644 --- a/litellm/llms/oobabooga/chat/oobabooga.py +++ b/litellm/llms/oobabooga/chat/oobabooga.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final import litellm from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.custom_httpx.http_handler import get_httpx_client +from litellm.llms.custom_httpx.sse_lines import iter_sse_lines from litellm.utils import EmbeddingResponse, ModelResponse, Usage from ..common_utils import OobaboogaError @@ -74,7 +75,7 @@ def completion( stream=optional_params["stream"] if "stream" in optional_params else False, ) if "stream" in optional_params and optional_params["stream"] is True: - return response.iter_lines() + return iter_sse_lines(response.iter_text()) else: return oobabooga_config.transform_response( model=model, diff --git a/litellm/llms/openai_like/chat/handler.py b/litellm/llms/openai_like/chat/handler.py index 2c74ef19e7d..a450947afdf 100644 --- a/litellm/llms/openai_like/chat/handler.py +++ b/litellm/llms/openai_like/chat/handler.py @@ -16,6 +16,7 @@ from litellm import LlmProviders from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines from litellm.llms.databricks.streaming_utils import ModelResponseIterator from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.openai.openai import OpenAIConfig @@ -65,7 +66,9 @@ async def make_call( model_response: Final = _fake_streamed_model_response(response.json()) completion_stream = MockResponseIterator(model_response=model_response) else: - completion_stream = ModelResponseIterator(streaming_response=response.aiter_lines(), sync_stream=False) + completion_stream = ModelResponseIterator( + streaming_response=aiter_sse_lines(response.aiter_text()), sync_stream=False + ) # LOGGING logging_obj.post_call( input=messages, @@ -103,7 +106,9 @@ def make_sync_call( model_response: Final = _fake_streamed_model_response(response.json()) completion_stream = MockResponseIterator(model_response=model_response) else: - completion_stream = ModelResponseIterator(streaming_response=response.iter_lines(), sync_stream=True) + completion_stream = ModelResponseIterator( + streaming_response=iter_sse_lines(response.iter_text()), sync_stream=True + ) # LOGGING logging_obj.post_call( diff --git a/litellm/llms/predibase/chat/handler.py b/litellm/llms/predibase/chat/handler.py index d0a61ea6e00..2ba4a63e08b 100644 --- a/litellm/llms/predibase/chat/handler.py +++ b/litellm/llms/predibase/chat/handler.py @@ -14,6 +14,7 @@ from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, get_async_httpx_client, ) +from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines from litellm.utils import CustomStreamWrapper, ModelResponse from ..common_utils import PredibaseError @@ -34,7 +35,7 @@ async def make_call( if response.status_code != 200: raise PredibaseError(status_code=response.status_code, message=response.text) - completion_stream: Final = response.aiter_lines() + completion_stream: Final = aiter_sse_lines(response.aiter_text()) # LOGGING logging_obj.post_call( input=messages, @@ -163,7 +164,7 @@ class PredibaseChatCompletion: timeout=timeout, ) _response: Final = CustomStreamWrapper( - response.iter_lines(), + iter_sse_lines(response.iter_text()), model, custom_llm_provider="predibase", logging_obj=logging_obj, diff --git a/litellm/llms/vertex_ai/agent_engine/transformation.py b/litellm/llms/vertex_ai/agent_engine/transformation.py index 9f592398f74..0339533e1f5 100644 --- a/litellm/llms/vertex_ai/agent_engine/transformation.py +++ b/litellm/llms/vertex_ai/agent_engine/transformation.py @@ -22,6 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, ) from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines from litellm.llms.vertex_ai.agent_engine.sse_iterator import ( VertexAgentEngineResponseIterator, ) @@ -357,7 +358,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): ) -> VertexAgentEngineResponseIterator: """Return a streaming iterator for SSE responses.""" return VertexAgentEngineResponseIterator( - streaming_response=raw_response.iter_lines(), + streaming_response=iter_sse_lines(raw_response.iter_text()), sync_stream=True, ) @@ -465,7 +466,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): # Create iterator for SSE stream (async) completion_stream: Final = VertexAgentEngineResponseIterator( - streaming_response=response.aiter_lines(), + streaming_response=aiter_sse_lines(response.aiter_text()), sync_stream=False, ) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index f27593dc94e..c506192f253 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -41,6 +41,7 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, get_httpx_client, ) +from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines from litellm.types.llms.anthropic import AnthropicThinkingParam from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.gemini import BidiGenerateContentServerMessage @@ -3241,7 +3242,7 @@ async def make_call( raise completion_stream: Final = ModelResponseIterator( - streaming_response=response.aiter_lines(), + streaming_response=aiter_sse_lines(response.aiter_text()), sync_stream=False, logging_obj=logging_obj, response_headers=response.headers, @@ -3312,7 +3313,7 @@ def make_sync_call( raise completion_stream: Final = ModelResponseIterator( - streaming_response=response.iter_lines(), + streaming_response=iter_sse_lines(response.iter_text()), sync_stream=True, logging_obj=logging_obj, response_headers=response.headers, @@ -4190,7 +4191,7 @@ class ModelResponseIterator: log_failed_hop(e) return self._truncated_stream_end(token) self._start_hop(hop_response, token) - self.streaming_response = hop_response.aiter_lines() + self.streaming_response = aiter_sse_lines(hop_response.aiter_text()) self.async_response_iterator = self.streaming_response return None @@ -4204,7 +4205,7 @@ class ModelResponseIterator: log_failed_hop(e) return self._truncated_stream_end(token) self._start_hop(hop_response, token) - self.streaming_response = hop_response.iter_lines() + self.streaming_response = iter_sse_lines(hop_response.iter_text()) self.response_iterator = self.streaming_response return None diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index 4691c7b0bc5..219ce906340 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -26,6 +26,7 @@ from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.url_utils import SSRFError, validate_url from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header +from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.a2a.version_convert import ( A2AVersion, @@ -320,7 +321,7 @@ async def _a2a_sse_event_source( "error": {"code": -32603, "message": resp.reason_phrase}, } return - async for line in resp.aiter_lines(): + async for line in aiter_sse_lines(resp.aiter_text()): stripped = line.strip() if not stripped.startswith("data:"): continue diff --git a/litellm/proxy/anthropic_endpoints/streaming_model_restamp.py b/litellm/proxy/anthropic_endpoints/streaming_model_restamp.py index f748a754fe0..32e0f6e8682 100644 --- a/litellm/proxy/anthropic_endpoints/streaming_model_restamp.py +++ b/litellm/proxy/anthropic_endpoints/streaming_model_restamp.py @@ -14,6 +14,8 @@ from typing import Final from pydantic import TypeAdapter, ValidationError +from litellm.llms.custom_httpx.sse_lines import split_sse_lines_keepends + _MESSAGE_START_EVENT: Final = "message_start" _MESSAGE_START_MARKER: Final = b"message_start" _SSE_DATA_FIELD: Final = "data:" @@ -52,7 +54,7 @@ def _restamped_data_line(line: str, requested_model: str) -> str | None: def _restamped_frame(frame: str, requested_model: str) -> str | None: - lines: Final = frame.splitlines(keepends=True) + lines: Final = split_sse_lines_keepends(frame) restamped: Final = tuple(_restamped_data_line(line, requested_model) for line in lines) if all(line is None for line in restamped): return None diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 7c7eb51bc5d..5415314f29b 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -29,6 +29,7 @@ from litellm.llms.anthropic.messages.stream_assembly import ( split_sse_chunk_into_events, ) from litellm.llms.bedrock.common_utils import BEDROCK_INVOKE_TRACE_FIELD +from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.proxy.auth.auth_utils import get_end_user_id_from_request_body from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import ( @@ -61,7 +62,7 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i def _event_payload(event: str) -> Mapping[str, object] | None: data_lines: Final = tuple( - line.strip()[len("data:") :].strip() for line in event.splitlines() if line.strip().startswith("data:") + line.strip()[len("data:") :].strip() for line in split_sse_lines(event) if line.strip().startswith("data:") ) if not data_lines or data_lines[0] in ("", "[DONE]"): return None @@ -362,7 +363,7 @@ class AnthropicPassthroughLoggingHandler: ) -> str | None: for raw in all_chunks: text = raw.decode("utf-8") if isinstance(raw, bytes) else raw - for line in text.splitlines(): + for line in split_sse_lines(text): if not line.startswith("data:"): continue try: @@ -402,7 +403,7 @@ class AnthropicPassthroughLoggingHandler: """ for raw in reversed(all_chunks): text = raw.decode("utf-8") if isinstance(raw, bytes) else raw - for line in reversed(text.splitlines()): + for line in reversed(split_sse_lines(text)): if not line.startswith("data:"): continue try: diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py index 2afc1d17bc2..0c4a3f64852 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py @@ -21,6 +21,7 @@ from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, get_async_httpx_client, ) +from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.passthrough_endpoints.tinyfish import ( @@ -81,7 +82,7 @@ def sse_poller_spawned(logging_obj: LiteLLMLoggingObj) -> bool: def run_id_from_sse_frames(frames: bytes) -> str | None: - return _run_id_from_sse_chunks(frames.decode("utf-8", errors="replace").splitlines()) + return _run_id_from_sse_chunks(split_sse_lines(frames.decode("utf-8", errors="replace"))) def resolve_tinyfish_agent_api_base() -> str: diff --git a/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py b/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py index 9790c150ae1..cae8ccc93ee 100644 --- a/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py +++ b/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py @@ -51,6 +51,7 @@ from litellm.llms.base_llm.managed_resources.isolation import ( can_access_resource, resolve_resource_owner_id, ) +from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames @@ -830,7 +831,7 @@ _SSE_EVENT_ADAPTER: Final = TypeAdapter(Mapping[str, JsonValue]) def _first_streamed_response(frames: bytes) -> tuple[str, Mapping[str, JsonValue]] | None: - for line in frames.decode("utf-8", errors="replace").splitlines(): + for line in split_sse_lines(frames.decode("utf-8", errors="replace")): if not line.startswith(_SSE_DATA_PREFIX): continue try: diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 5f6a7432afb..1eb76f8f558 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -18,6 +18,7 @@ from starlette.types import Message from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger +from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler @@ -81,7 +82,7 @@ class _StreamsBody(Protocol): def _sse_frame_data(frame: str) -> str | None: - return next((line[6:].strip() for line in frame.splitlines() if line.startswith("data: ")), None) + return next((line[6:].strip() for line in split_sse_lines(frame) if line.startswith("data: ")), None) async def _never_receive() -> Message: diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 219944de521..982158040c4 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -165,6 +165,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._upstream_exhausted: bool = False self._response_id_primed: bool = False self._pending_tool_events: list[BaseLiteLLMOpenAIResponseObject] = [] + self._tool_calls_held_behind_text: tuple[object, ...] = () self._tool_output_index_by_call_id: dict[str, int] = {} self._tool_args_by_call_id: dict[str, str] = {} self._tool_item_id_by_call_id: dict[str, str] = {} # mutable-ok: filled per call id as tool call events stream @@ -388,6 +389,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return "" return self._tool_call_id_by_index.get(index, "") + def _hold_tool_calls_behind_text(self, tool_calls: object) -> None: + if isinstance(tool_calls, list): + self._tool_calls_held_behind_text = (*self._tool_calls_held_behind_text, *tool_calls) + + def _release_held_tool_calls(self) -> tuple[object, ...]: + held: Final = self._tool_calls_held_behind_text + self._tool_calls_held_behind_text = () + return held + def _queue_tool_call_delta_events(self, tool_calls: object) -> None: """ Convert chat-completions streaming `tool_calls` deltas into Responses API streaming events. @@ -1365,6 +1375,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if delta_content: if not self.sent_message_item_added_event: self._queue_message_item_added_events() + self._hold_tool_calls_behind_text(getattr(chunk.choices[0].delta, "tool_calls", None)) self._sequence_number += 1 text_delta_event: Final = OutputTextDeltaEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, @@ -1379,7 +1390,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): # Priority 3: Handle tool call deltas (if any) -> queue events and emit them # For each tool call delta, we emit events one at a time to match OpenAI's streaming behavior if chunk.choices and hasattr(chunk.choices[0].delta, "tool_calls") and chunk.choices[0].delta.tool_calls: - self._queue_tool_call_delta_events(chunk.choices[0].delta.tool_calls) + self._queue_tool_call_delta_events([*self._release_held_tool_calls(), *chunk.choices[0].delta.tool_calls]) # Return one pending tool event at a time if self._pending_tool_events: return self._pending_tool_events.pop(0) diff --git a/tests/benchmarks/test_sse_lines_benchmarks.py b/tests/benchmarks/test_sse_lines_benchmarks.py new file mode 100644 index 00000000000..1320fae49a6 --- /dev/null +++ b/tests/benchmarks/test_sse_lines_benchmarks.py @@ -0,0 +1,27 @@ +from collections.abc import Sequence +from typing import Final + +import pytest + +from litellm.llms.custom_httpx.sse_lines import iter_sse_lines + +EVENT: Final = 'data: {"choices":[{"delta":{"content":"hello world"}}]}' + + +def _chunks(line_break: str) -> Sequence[str]: + body: Final = "".join(f"{EVENT}{line_break}{line_break}" for _ in range(2000)) + return tuple(body[i : i + 512] for i in range(0, len(body), 512)) + + +LF_CHUNKS: Final = _chunks("\n") +CRLF_CHUNKS: Final = _chunks("\r\n") + + +@pytest.mark.benchmark +def test_iter_sse_lines_lf_stream() -> None: + assert sum(1 for _ in iter_sse_lines(LF_CHUNKS)) == 4000 + + +@pytest.mark.benchmark +def test_iter_sse_lines_crlf_stream() -> None: + assert sum(1 for _ in iter_sse_lines(CRLF_CHUNKS)) == 4000 diff --git a/tests/unit/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py b/tests/unit/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py index 09f09170241..cd3f7622fa2 100644 --- a/tests/unit/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py +++ b/tests/unit/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py @@ -10,6 +10,7 @@ Verifies that: """ import json +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -560,6 +561,37 @@ class TestStreaming: assert sent_body["params"]["message"]["messageId"] == "msg-001" assert [event["result"]["kind"] for event in events] == ["task", "status-update"] + @pytest.mark.asyncio + @pytest.mark.usefixtures("httpx_transport") + async def test_streaming_keeps_unicode_line_separators_inside_an_sse_event(self) -> None: + from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, + ) + + event: Final = { + "jsonrpc": "2.0", + "id": "req-001", + "result": {"kind": "artifact-update", "artifact": {"parts": [{"kind": "text", "text": "a\u2028b\u2029c"}]}}, + } + with respx.mock(assert_all_called=True) as router: + router.post(url__regex=r".*/invocations.*").mock( + return_value=httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text=f"data: {json.dumps(event, ensure_ascii=False)}\n\n", + ) + ) + events: Final = [ + streamed + async for streamed in BedrockAgentCoreA2AConfig().handle_streaming( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + ) + ] + + assert events == [event] + class TestConfigManager: """Test that config manager routes 'bedrock' correctly.""" diff --git a/tests/unit/a2a_protocol/providers/watsonx_orchestrate/test_watsonx_orchestrate_transformation.py b/tests/unit/a2a_protocol/providers/watsonx_orchestrate/test_watsonx_orchestrate_transformation.py index a300560ae9d..ac3f0390f84 100644 --- a/tests/unit/a2a_protocol/providers/watsonx_orchestrate/test_watsonx_orchestrate_transformation.py +++ b/tests/unit/a2a_protocol/providers/watsonx_orchestrate/test_watsonx_orchestrate_transformation.py @@ -1,6 +1,7 @@ import asyncio import json import time +from typing import Final import httpx import pytest @@ -36,13 +37,8 @@ class _ShortTtlTokenClient: return _JsonResponse({"access_token": f"token-{self.calls}", "expires_in": 30}) -class _SSELines: - def __init__(self, lines): - self.lines = lines - - async def aiter_lines(self): - for line in self.lines: - yield line +def _sse_response(lines: list[str]) -> httpx.Response: + return httpx.Response(200, stream=httpx.ByteStream("".join(f"{line}\n" for line in lines).encode())) class _InvalidJsonStreamResponse: @@ -218,7 +214,7 @@ def test_cp4d_token_ttl_from_absolute_expiration(): @pytest.mark.asyncio async def test_accumulate_wxo_sse_text_ignores_non_dict_json_events(): - response = _SSELines( + response = _sse_response( [ "data: null", "data: true", @@ -230,6 +226,14 @@ async def test_accumulate_wxo_sse_text_ignores_non_dict_json_events(): ) +@pytest.mark.asyncio +async def test_accumulate_wxo_sse_text_keeps_unicode_line_separators_inside_an_event() -> None: + separated_text: Final = "before\u2028between\u2029after" + response: Final = _sse_response([f"data: {json.dumps({'results': separated_text}, ensure_ascii=False)}", ""]) + + assert await WatsonxOrchestrateHandler._accumulate_wxo_sse_text(response) == separated_text + + @pytest.mark.asyncio async def test_short_lived_tokens_are_not_served_from_cache(): client = _ShortTtlTokenClient() diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 9aa494c1aab..46f1dbf2e7a 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1102,6 +1102,24 @@ def test_recover_output_items_merges_text_only_items_at_distinct_indices(): assert recovered[1]["content"][0]["text"] == "From OUTPUT_TEXT_DONE only" +def test_recover_output_items_keeps_an_item_whose_text_holds_unicode_line_separators() -> None: + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + item: Final = { + "type": "message", + "id": "msg_item_0", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "From\u2028OUTPUT_ITEM\u2029DONE", "annotations": []}], + } + event: Final = {"type": "response.output_item.done", "output_index": 0, "item": item} + raw_sse: Final = f"data: {json.dumps(event, ensure_ascii=False)}\ndata: [DONE]\n" + + assert LiteLLMResponsesTransformationHandler._recover_output_items_from_raw_sse(raw_sse) == [item] + + def test_transform_response_prefers_completed_output_from_raw_sse(): from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, diff --git a/tests/unit/google_genai/test_google_genai_streaming_iterator.py b/tests/unit/google_genai/test_google_genai_streaming_iterator.py index 120c99b6e42..cbb9638b3d9 100644 --- a/tests/unit/google_genai/test_google_genai_streaming_iterator.py +++ b/tests/unit/google_genai/test_google_genai_streaming_iterator.py @@ -62,7 +62,9 @@ def _logging_obj( ], ) def test_streaming_logging_targets_the_provider_that_served_the_request( - iterator_cls: type, + iterator_cls: ( + type[AsyncGoogleGenAIGenerateContentStreamingIterator] | type[GoogleGenAIGenerateContentStreamingIterator] + ), custom_llm_provider: str, expected_endpoint_type: EndpointType, ): @@ -99,18 +101,15 @@ def _large_inline_data_event() -> str: return f"data: {json.dumps(payload)}" +def _sse_response(body: str) -> httpx.Response: + return httpx.Response(200, stream=httpx.ByteStream(body.encode())) + + @pytest.mark.asyncio async def test_async_streaming_iterator_yields_complete_sse_events(): """Large inlineData must not be split across byte-chunk boundaries.""" - mock_response = MagicMock() - - async def _aiter_lines(): - yield _large_inline_data_event() - - mock_response.aiter_lines = _aiter_lines - iterator = AsyncGoogleGenAIGenerateContentStreamingIterator( - response=mock_response, + response=_sse_response(_large_inline_data_event() + "\n\n"), model="gemini-3.1-flash-image-preview", logging_obj=_logging_obj(), generate_content_provider_config=MagicMock(), @@ -128,11 +127,8 @@ async def test_async_streaming_iterator_yields_complete_sse_events(): def test_sync_streaming_iterator_yields_complete_sse_events(): - mock_response = MagicMock() - mock_response.iter_lines.return_value = iter([_large_inline_data_event()]) - iterator = GoogleGenAIGenerateContentStreamingIterator( - response=mock_response, + response=_sse_response(_large_inline_data_event() + "\n\n"), model="gemini-3.1-flash-image-preview", logging_obj=_logging_obj(), generate_content_provider_config=MagicMock(), @@ -150,17 +146,8 @@ def test_sync_streaming_iterator_yields_complete_sse_events(): @pytest.mark.asyncio async def test_async_streaming_iterator_preserves_multi_field_sse_event(): - mock_response = MagicMock() - - async def _aiter_lines(): - yield "event: message" - yield 'data: {"text":"hi"}' - yield "" - - mock_response.aiter_lines = _aiter_lines - iterator = AsyncGoogleGenAIGenerateContentStreamingIterator( - response=mock_response, + response=_sse_response('event: message\ndata: {"text":"hi"}\n\n'), model="gemini-test", logging_obj=_logging_obj(), generate_content_provider_config=MagicMock(), @@ -174,16 +161,8 @@ async def test_async_streaming_iterator_preserves_multi_field_sse_event(): @pytest.mark.asyncio async def test_async_streaming_iterator_forwards_sse_comment_events(): - mock_response = MagicMock() - - async def _aiter_lines(): - yield ": keepalive" - yield "" - - mock_response.aiter_lines = _aiter_lines - iterator = AsyncGoogleGenAIGenerateContentStreamingIterator( - response=mock_response, + response=_sse_response(": keepalive\n\n"), model="gemini-test", logging_obj=_logging_obj(), generate_content_provider_config=MagicMock(), @@ -195,6 +174,46 @@ async def test_async_streaming_iterator_forwards_sse_comment_events(): assert chunk == b": keepalive\n\n" +_LINE_SEPARATED_TEXT: Final = "before\u2028between\u2029after" + + +async def _first_chunk( + iterator: GoogleGenAIGenerateContentStreamingIterator | AsyncGoogleGenAIGenerateContentStreamingIterator, +) -> bytes: + if isinstance(iterator, AsyncGoogleGenAIGenerateContentStreamingIterator): + return await iterator.__anext__() + return next(iterator) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "iterator_cls", + [AsyncGoogleGenAIGenerateContentStreamingIterator, GoogleGenAIGenerateContentStreamingIterator], +) +async def test_streaming_iterator_keeps_unicode_line_separators_inside_the_event( + iterator_cls: ( + type[AsyncGoogleGenAIGenerateContentStreamingIterator] | type[GoogleGenAIGenerateContentStreamingIterator] + ), +) -> None: + payload: Final = {"candidates": [{"content": {"parts": [{"text": _LINE_SEPARATED_TEXT}]}}]} + event: Final = f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" + iterator: Final = iterator_cls( + response=_sse_response(event), + model="gemini-3.8-flash", + logging_obj=_logging_obj(), + generate_content_provider_config=MagicMock(), + litellm_metadata={}, + custom_llm_provider="gemini", + ) + + chunk: Final = await _first_chunk(iterator) + + assert chunk == event.encode() + assert ( + json.loads(chunk[len(b"data: ") : -2])["candidates"][0]["content"]["parts"][0]["text"] == _LINE_SEPARATED_TEXT + ) + + def _gemini_event(text: str, finish_reason: str | None = None) -> str: candidate: Final = {"content": {"role": "model", "parts": [{"text": text}]}, "index": 0} payload: Final = { diff --git a/tests/unit/interactions/test_interactions_streaming_iterator.py b/tests/unit/interactions/test_interactions_streaming_iterator.py index afcd0d559d7..fb1fe92d637 100644 --- a/tests/unit/interactions/test_interactions_streaming_iterator.py +++ b/tests/unit/interactions/test_interactions_streaming_iterator.py @@ -16,7 +16,10 @@ import pytest import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.interactions import streaming_iterator as interactions_streaming_iterator_module -from litellm.interactions.streaming_iterator import InteractionsAPIStreamingIterator +from litellm.interactions.streaming_iterator import ( + InteractionsAPIStreamingIterator, + SyncInteractionsAPIStreamingIterator, +) from litellm.litellm_core_utils import thread_pool_executor as thread_pool_executor_module from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig @@ -148,3 +151,43 @@ async def test_stream_logs_its_first_event_time_measured_from_the_call_start(): assert first_event_at is not None assert recorder.logged == [(call_start, first_event_at)] + + +_LINE_SEPARATED_TEXT: Final = "before\u2028between\u2029after" + + +async def _first_event( + iterator: InteractionsAPIStreamingIterator | SyncInteractionsAPIStreamingIterator, +) -> InteractionsAPIStreamingResponse: + if isinstance(iterator, InteractionsAPIStreamingIterator): + return await iterator.__anext__() + return next(iterator) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("iterator_cls", [InteractionsAPIStreamingIterator, SyncInteractionsAPIStreamingIterator]) +async def test_stream_keeps_unicode_line_separators_inside_the_delta_text( + iterator_cls: type[InteractionsAPIStreamingIterator] | type[SyncInteractionsAPIStreamingIterator], +) -> None: + delta: Final = {"type": "text", "text": _LINE_SEPARATED_TEXT} + event: Final = {"event_type": "step.delta", "id": "interaction-1", "delta": delta} + iterator: Final = iterator_cls( + response=httpx.Response( + 200, stream=httpx.ByteStream(f"data: {json.dumps(event, ensure_ascii=False)}\n\n".encode()) + ), + model="gemini/gemini-3.1-pro-preview", + interactions_api_config=GoogleAIStudioInteractionsConfig(), + logging_obj=LitellmLogging( + model="gemini/gemini-3.1-pro-preview", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="ainteraction", + start_time=datetime(2026, 10, 9, 12, 0, 0), + litellm_call_id="interactions-separators", + function_id="interactions-separators", + ), + ) + + first: Final = await _first_event(iterator) + + assert first.delta == delta diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index a1b58521d7e..79978ecfe8e 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -506,6 +506,118 @@ def test_multi_chunk_reasoning_and_content( assert initialized_custom_stream_wrapper.sent_last_thinking_block is True +def _stream_chunk(delta: dict[str, object], finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream(choices=[{"index": 0, "delta": delta, "finish_reason": finish_reason}]) + + +def _content_through_first_call(deltas: Sequence[Delta]) -> str: + first_call: Final = next( + (index for index, delta in enumerate(deltas) if delta.tool_calls or delta.function_call), len(deltas) + ) + return "".join(delta.content or "" for delta in deltas[: first_call + 1]) + + +_GET_WEATHER_CALL: Final = { + "index": 0, + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [True, False]) +@pytest.mark.parametrize( + "chunks,expected_content,expected_tool_calls", + [ + ( + ( + _stream_chunk({"reasoning_content": "Check the weather"}), + _stream_chunk({"tool_calls": [_GET_WEATHER_CALL]}), + _stream_chunk({}, "tool_calls"), + ), + "Check the weather", + 1, + ), + ( + ( + _stream_chunk({"reasoning_content": "Pick a function"}), + _stream_chunk({"function_call": {"name": "get_weather", "arguments": "{}"}}), + _stream_chunk({}, "function_call"), + ), + "Pick a function", + 0, + ), + ( + ( + _stream_chunk({"reasoning_content": "Step one"}), + _stream_chunk({"reasoning_content": ", step two"}), + _stream_chunk({}, "length"), + ), + "Step one, step two", + 0, + ), + ( + ( + _stream_chunk({"reasoning_content": "a"}), + _stream_chunk({"content": "b"}), + _stream_chunk({"reasoning_content": "c"}), + _stream_chunk({"content": "d"}), + _stream_chunk({}, "stop"), + ), + "abcd", + 0, + ), + ( + (_stream_chunk({"reasoning_content": "Still thinking"}),), + "Still thinking", + 0, + ), + ( + (_stream_chunk({"content": "hi"}), _stream_chunk({}, "stop")), + "hi", + 0, + ), + ], + ids=[ + "reasoning_then_tool_call", + "reasoning_then_function_call", + "reasoning_cut_by_length", + "reasoning_after_answer", + "stream_ends_mid_reasoning", + "answer_without_reasoning", + ], +) +async def test_merged_reasoning_stream_keeps_think_tags_balanced( + logging_obj: Logging, + is_async: bool, + chunks: tuple[ModelResponseStream, ...], + expected_content: str, + expected_tool_calls: int, +) -> None: + response: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator(model_responses=[chunk.model_copy(deep=True) for chunk in chunks]), + model="deepseek-reasoner", + logging_obj=logging_obj, + ) + response.merge_reasoning_content_in_choices = True + + deltas: Final = ( + tuple([chunk.choices[0].delta async for chunk in response]) + if is_async + else tuple(chunk.choices[0].delta for chunk in response) + ) + logged: Final = litellm.stream_chunk_builder(chunks=response.chunks) + + assert "".join(delta.content or "" for delta in deltas) == expected_content + assert _content_through_first_call(deltas) == expected_content + assert sum(len(delta.tool_calls or ()) for delta in deltas) == expected_tool_calls + assert logged.choices[0].message.content == (expected_content or None) + assert any(chunk.choices[0].finish_reason for chunk in response.chunks if chunk.choices) == any( + chunk.choices[0].finish_reason for chunk in chunks + ) + + def test_strip_sse_data_from_chunk(): """Test the static method that strips 'data: ' prefix from SSE chunks""" # Test with string inputs @@ -7849,7 +7961,7 @@ def gemini_mock_post_streaming(url, **kwargs): mock_response = MagicMock() mock_response.status_code = 200 mock_response.headers = {"Content-Type": "text/event-stream"} - mock_response.iter_lines = MagicMock(return_value=stream_response()) + mock_response.iter_text = MagicMock(return_value=stream_response()) return mock_response diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index 1f4a5aaa256..355d407001c 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -15,7 +15,7 @@ import litellm from litellm._uuid import uuid from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt -from litellm.llms.anthropic.chat.handler import ModelResponseIterator, _envelope_verdict, make_call +from litellm.llms.anthropic.chat.handler import ModelResponseIterator, _envelope_verdict, make_call, make_sync_call from litellm.llms.base_llm.base_utils import response_format_schema, unwrap_response_format_envelope from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.llms.openai import ( @@ -27,24 +27,34 @@ from litellm.types.utils import ChatCompletionDeltaToolCall, ModelResponseStream from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE +def _text_delta_stream(text: str) -> bytes: + events: Final = ( + ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}), + ) + return "".join( + f"event: {name}\ndata: {json.dumps(payload, ensure_ascii=False)}\n\n" for name, payload in events + ).encode() + + +def _streamed_text(chunks: list[object]) -> str: + return "".join(chunk.choices[0].delta.content or "" for chunk in chunks if isinstance(chunk, ModelResponseStream)) + + @pytest.mark.asyncio async def test_make_call_passes_logging_obj_to_client_post(): """make_call must pass logging_obj to client.post so track_llm_api_timing can set llm_api_duration_ms for litellm_overhead_time_ms.""" mock_client = AsyncMock() - mock_response = MagicMock() - mock_response.aiter_lines = MagicMock( - return_value=iter([b'data: {"type":"message_start"}\n', b'data: {"type":"message_delta"}\n']) - ) - mock_client.post.return_value = mock_response + mock_client.post.return_value = httpx.Response(200, stream=httpx.ByteStream(_text_delta_stream("Hi there"))) logging_obj = MagicMock() - await make_call( + completion_stream, _ = await make_call( client=mock_client, api_base="https://api.anthropic.com/v1/messages", headers={}, data="{}", - model="claude-3-5-haiku", + model="claude-haiku-5-5", messages=[{"role": "user", "content": "Hi"}], logging_obj=logging_obj, timeout=60.0, @@ -54,6 +64,39 @@ async def test_make_call_passes_logging_obj_to_client_post(): mock_client.post.assert_called_once() call_kwargs = mock_client.post.call_args[1] assert call_kwargs.get("logging_obj") is logging_obj + assert _streamed_text([chunk async for chunk in completion_stream]) == "Hi there" + + +_LINE_SEPARATED_TEXT: Final = "before\u2028between\u2029after" + + +async def _streamed_chunks(sync_stream: bool, text: str) -> list[object]: + transport: Final = httpx.MockTransport( + lambda request: httpx.Response(200, stream=httpx.ByteStream(_text_delta_stream(text))) + ) + call_args: Final = { + "api_base": "https://api.anthropic.com/v1/messages", + "headers": {}, + "data": "{}", + "model": "claude-haiku-5-5", + "messages": [{"role": "user", "content": "Hi"}], + "logging_obj": MagicMock(), + "timeout": 60.0, + "json_mode": False, + } + if sync_stream: + sync_iterator, _ = make_sync_call(client=HTTPHandler(client=httpx.Client(transport=transport)), **call_args) + return list(sync_iterator) + async_iterator, _ = await make_call(client=AsyncHTTPHandler(transport=transport), **call_args) + return [chunk async for chunk in async_iterator] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_stream", [True, False]) +async def test_streamed_text_keeps_unicode_line_separators(sync_stream: bool) -> None: + chunks: Final = await _streamed_chunks(sync_stream, _LINE_SEPARATED_TEXT) + + assert _streamed_text(chunks) == _LINE_SEPARATED_TEXT def test_anthropic_completion_does_not_send_deployment_default_limits(): diff --git a/tests/unit/llms/anthropic/messages/test_stream_assembly.py b/tests/unit/llms/anthropic/messages/test_stream_assembly.py index f7d6b8d4f8c..4993d2fa9bf 100644 --- a/tests/unit/llms/anthropic/messages/test_stream_assembly.py +++ b/tests/unit/llms/anthropic/messages/test_stream_assembly.py @@ -144,6 +144,30 @@ def test_anthropic_message_delta_fields_reads_the_fields_the_openai_shape_drops( assert dict(anthropic_message_delta_fields(GOOGLE_STREAM)) == {} +def test_anthropic_message_delta_fields_reads_a_message_delta_whose_json_holds_unicode_line_separators() -> None: + safeguard_results: Final = [ + { + "type": "dangerous_tool_use", + "status": { + "type": "available", + "tool_uses": { + "toolu_01": {"type": "evaluated", "outcome": "flagged", "explanation": "deletes\u2028the\u2029root"} + }, + }, + } + ] + message_delta: Final = { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "safeguard_results": safeguard_results}, + "usage": {"output_tokens": 2}, + } + raw_message_delta_frame: Final = f"event: message_delta\ndata: {json.dumps(message_delta, ensure_ascii=False)}\n\n" + + fields: Final = anthropic_message_delta_fields((*ANTHROPIC_STREAM[:5], raw_message_delta_frame.encode())) + + assert dict(fields) == {"safeguard_results": safeguard_results} + + @pytest.mark.parametrize( "chunks", [GOOGLE_STREAM, INVALID_UTF8, (ANTHROPIC_ERROR,), ()], diff --git a/tests/unit/llms/azure_ai/agents/test_handler.py b/tests/unit/llms/azure_ai/agents/test_handler.py index 71466bb172c..14b6094747f 100644 --- a/tests/unit/llms/azure_ai/agents/test_handler.py +++ b/tests/unit/llms/azure_ai/agents/test_handler.py @@ -173,7 +173,7 @@ async def test_handler_acompletion_with_thread_id_reuses_that_thread() -> None: def _sse(event: str, data: dict[str, object] | str) -> str: - return f"event: {event}\ndata: {data if isinstance(data, str) else json.dumps(data)}\n\n" + return f"event: {event}\ndata: {data if isinstance(data, str) else json.dumps(data, ensure_ascii=False)}\n\n" def _message_delta(text: str) -> dict[str, object]: @@ -222,3 +222,24 @@ async def test_acompletion_stream_yields_agent_message_deltas_then_stop(respx_mo "thread": {"messages": [{"role": "user", "content": "Hi Agent, what is 10 + 5?"}]}, } assert chunks == [("10 + 5", None), (" = 15", None), (None, "stop")] + + +@pytest.mark.asyncio +async def test_acompletion_stream_keeps_unicode_line_separators_inside_a_delta(respx_mock: respx.MockRouter) -> None: + separated_text: Final = "before\u2028between\u2029after" + _ = respx_mock.post(f"{API_BASE}/threads/runs").respond( + 200, + text=_sse("thread.message.delta", _message_delta(separated_text)) + _sse("done", "[DONE]"), + headers={"content-type": "text/event-stream"}, + ) + + response: Final = await litellm.acompletion( + model=f"azure_ai/agents/{AGENT_ID}", + messages=[{"role": "user", "content": "Say it with separators"}], + api_base=API_BASE, + api_key="azure-ad-token", + stream=True, + ) + chunks: Final = [(chunk.choices[0].delta.content, chunk.choices[0].finish_reason) async for chunk in response] + + assert chunks == [(separated_text, None), (None, "stop")] diff --git a/tests/unit/llms/azure_ai/agents/test_transformation.py b/tests/unit/llms/azure_ai/agents/test_transformation.py index 10676438858..413908c8313 100644 --- a/tests/unit/llms/azure_ai/agents/test_transformation.py +++ b/tests/unit/llms/azure_ai/agents/test_transformation.py @@ -1,7 +1,7 @@ import json from typing import Final -from unittest.mock import MagicMock +import httpx import pytest from litellm.llms.azure_ai.agents.transformation import AzureAIAgentsConfig @@ -428,15 +428,10 @@ async def test_azure_ai_agents_streaming_accumulates_annotations_from_multiple_t "data: [DONE]", ] - async def mock_aiter_lines(): - for line in sse_lines: - yield line - - mock_response = MagicMock() - mock_response.aiter_lines = MagicMock(return_value=mock_aiter_lines()) + sse_response = httpx.Response(200, stream=httpx.ByteStream("\n".join(sse_lines).encode())) chunks = [] - async for chunk in handler._process_sse_stream(mock_response, "azure_ai/agents/asst_123"): + async for chunk in handler._process_sse_stream(sse_response, "azure_ai/agents/asst_123"): chunks.append(chunk) final_chunk = chunks[-1] @@ -498,15 +493,10 @@ async def test_azure_ai_agents_streaming_annotations_from_completed_message(): "data: [DONE]", ] - async def mock_aiter_lines(): - for line in sse_lines: - yield line - - mock_response = MagicMock() - mock_response.aiter_lines = MagicMock(return_value=mock_aiter_lines()) + sse_response = httpx.Response(200, stream=httpx.ByteStream("\n".join(sse_lines).encode())) chunks = [] - async for chunk in handler._process_sse_stream(mock_response, "azure_ai/agents/asst_123"): + async for chunk in handler._process_sse_stream(sse_response, "azure_ai/agents/asst_123"): chunks.append(chunk) assert len(chunks) >= 1 diff --git a/tests/unit/llms/base_llm/guardrail_translation/test_utils.py b/tests/unit/llms/base_llm/guardrail_translation/test_utils.py index 65208a2bd3b..acd50616f90 100644 --- a/tests/unit/llms/base_llm/guardrail_translation/test_utils.py +++ b/tests/unit/llms/base_llm/guardrail_translation/test_utils.py @@ -1,9 +1,11 @@ +import json from typing import Final import pytest import litellm from litellm.llms.base_llm.guardrail_translation.utils import ( + blocked_response_usage, effective_skip_system_message_for_guardrail, effective_skip_tool_message_for_guardrail, ) @@ -32,3 +34,18 @@ def test_a_per_guardrail_skip_flag_wins_over_the_global_setting( assert effective_skip_system_message_for_guardrail(guardrail) is expected assert effective_skip_tool_message_for_guardrail(guardrail) is expected + + +def test_blocked_response_usage_reads_a_message_delta_whose_json_holds_unicode_line_separators() -> None: + message_start: Final = {"type": "message_start", "message": {"usage": {"input_tokens": 11, "output_tokens": 1}}} + message_delta: Final = { + "type": "message_delta", + "delta": {"stop_reason": "stop_sequence", "stop_sequence": "END\u2028OF\u2029TURN"}, + "usage": {"output_tokens": 7}, + } + stream: Final = "".join( + f"event: {event['type']}\ndata: {json.dumps(event, ensure_ascii=False)}\n\n" + for event in (message_start, message_delta) + ).encode() + + assert blocked_response_usage([stream]) == {"input_tokens": 11, "output_tokens": 7} diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 7e05ef500f5..9d9e39df944 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -4458,6 +4458,20 @@ class TestMalformedBatchJsonl: _VALID_BATCH_RECORD: Final = json.loads(_VALID_BATCH_LINE) +def test_file_content_conversion_keeps_a_line_whose_json_holds_unicode_line_separators() -> None: + text: Final = "Hello\u2028from\u2029batch" + body: Final = {**_VALID_BATCH_RECORD["body"], "messages": [{"role": "user", "content": text}]} + record: Final = {**_VALID_BATCH_RECORD, "body": body} + file_content: Final = f"{json.dumps(record, ensure_ascii=False)}\n".encode() + + bedrock_jsonl, _ = BedrockJsonlFilesTransformation().transform_openai_file_content_to_bedrock_file_content( + file_content + ) + + (bedrock_record,) = (json.loads(line) for line in bedrock_jsonl.split("\n")) + assert bedrock_record["modelInput"]["messages"] == [{"role": "user", "content": [{"text": text}]}] + + def _batch_line(body: object) -> str: return json.dumps({**_VALID_BATCH_RECORD, "body": body}) diff --git a/tests/unit/llms/bedrock/passthrough/guardrail_translation/test_handler.py b/tests/unit/llms/bedrock/passthrough/guardrail_translation/test_handler.py index 5aaee1f0ee4..aec1672b335 100644 --- a/tests/unit/llms/bedrock/passthrough/guardrail_translation/test_handler.py +++ b/tests/unit/llms/bedrock/passthrough/guardrail_translation/test_handler.py @@ -18,7 +18,7 @@ import types from collections.abc import Callable, Mapping, Sequence from datetime import datetime from itertools import chain -from typing import ClassVar, Literal, TypeAlias +from typing import ClassVar, Final, Literal, TypeAlias from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -60,6 +60,7 @@ from litellm.llms.bedrock.passthrough.guardrail_translation.handler import ( BedrockPassthroughGuardrailHandler, _extract_converse_texts, _is_converse_endpoint, + _sse_events, _write_back_texts, ) @@ -3840,3 +3841,14 @@ class TestInvokeStreamGuardrailRelay: assert received_key is not None and received_key.texts == ("In Paris",) assert rewritten_key == received_key + + +def test_sse_events_keeps_a_data_line_whose_json_holds_unicode_line_separators() -> None: + event: Final = { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "Card\u2028on\u2029file"}, + } + sse: Final = f"event: content_block_delta\ndata: {json.dumps(event, ensure_ascii=False)}\n\n".encode() + + assert _sse_events(sse) == (event,) diff --git a/tests/unit/llms/bedrock/search/test_agentcore_search_transformation.py b/tests/unit/llms/bedrock/search/test_agentcore_search_transformation.py index 664dbf87fe9..01cca1fc270 100644 --- a/tests/unit/llms/bedrock/search/test_agentcore_search_transformation.py +++ b/tests/unit/llms/bedrock/search/test_agentcore_search_transformation.py @@ -8,6 +8,7 @@ the sharded CI (coverage collection runs against this tree). import json import os +from typing import Final import httpx import pytest @@ -682,3 +683,17 @@ def test_transform_search_response_ignores_text_blocks_without_result_objects(bl body = {"result": {"content": [{"type": "text", "text": block_text}]}} assert _transform_text(json.dumps(body)).results == [] + + +def test_transform_search_response_keeps_a_result_whose_sse_json_holds_unicode_line_separators() -> None: + snippet: Final = "Snippet\u2028for\u2029result" + results: Final = [{"title": "Result", "url": "https://example.com/1", "text": snippet}] + body: Final = { + "jsonrpc": "2.0", + "id": 1, + "result": {"content": [{"type": "text", "text": json.dumps(results, ensure_ascii=False)}]}, + } + + response: Final = _transform_text(f"event: message\ndata: {json.dumps(body, ensure_ascii=False)}\n\n") + + assert _as_tuples(response) == [("Result", "https://example.com/1", snippet, None, None)] diff --git a/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index a6f7d613f28..635363e77ea 100644 --- a/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -587,6 +587,37 @@ class TestChatGPTResponsesMultiAccount: assert headers_default["ChatGPT-Account-Id"] == "acct-default" +def test_chatgpt_non_stream_sse_keeps_an_output_item_whose_text_holds_unicode_line_separators() -> None: + text: Final = "Hello\u2028from\u2029stream" + output_item_done: Final = { + "type": "response.output_item.done", + "output_index": 0, + "item": {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": text}]}, + } + completed: Final = { + "type": "response.completed", + "response": { + "id": "resp_test", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-6.1-sol", + "output": [], + }, + } + sse_body: Final = "".join( + f"data: {json.dumps(event, ensure_ascii=False)}\n" for event in (output_item_done, completed) + ) + + parsed: Final = ChatGPTResponsesAPIConfig().transform_response_api_response( + model="chatgpt/gpt-6.1-sol", + raw_response=httpx.Response(200, headers={"content-type": "text/event-stream"}, text=sse_body), + logging_obj=MagicMock(), + ) + + assert parsed.output_text == text + + @pytest.mark.parametrize("call", ["responses", "completion"]) def test_prompt_cache_key_keeps_one_session_header_across_requests( call: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/llms/cloudflare/test_cloudflare_transformation.py b/tests/unit/llms/cloudflare/test_cloudflare_transformation.py index 3a45c83556c..ccbfc02f0e6 100644 --- a/tests/unit/llms/cloudflare/test_cloudflare_transformation.py +++ b/tests/unit/llms/cloudflare/test_cloudflare_transformation.py @@ -1,6 +1,6 @@ import asyncio import json -from collections.abc import Callable, Iterator +from collections.abc import AsyncIterator, Callable, Iterator from typing import Final from unittest.mock import MagicMock @@ -380,6 +380,35 @@ def test_completion_cloudflare_stream(sync_mode: bool, _cloudflare_httpx_transpo assert "language" in content.lower() +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"]) +async def test_cloudflare_stream_keeps_unicode_line_separators_inside_content( + sync_mode: bool, _cloudflare_httpx_transport: None +) -> None: + streamed_text: Final = ("alpha\u2028", "beta\u2029", "gamma") + base: Final = {"id": "chatcmpl-cf", "object": "chat.completion.chunk", "created": 1, "model": "@cf/glm"} + body: Final = "".join( + f"data: {json.dumps({**base, 'choices': [{'index': 0, 'delta': {'content': text}}]}, ensure_ascii=False)}\n\n" + for text in streamed_text + ) + with respx.mock(assert_all_called=True) as api: + api.post(f"{FAKE_API_BASE}/chat/completions").respond( + 200, headers={"content-type": "text/event-stream"}, content=(body + "data: [DONE]\n\n").encode() + ) + request: Final = { + "model": "cloudflare/@cf/glm", + "messages": [{"role": "user", "content": "repeat"}], + "stream": True, + "api_base": FAKE_API_BASE, + "api_key": FAKE_API_KEY, + } + chunks: Final = ( + tuple(completion(**request)) if sync_mode else tuple([c async for c in await acompletion(**request)]) + ) + + assert "".join(c.choices[0].delta.content or "" for c in chunks) == "".join(streamed_text) + + def _make_mock_response(json_data: Dict[str, Any]) -> MagicMock: mock = MagicMock(spec=httpx.Response) mock.status_code = 200 @@ -397,13 +426,13 @@ def test_acompletion_cloudflare_stream(sync_mode, monkeypatch): if sync_mode: - def _iter_lines(): + def _iter_text() -> Iterator[str]: for chunk in raw_chunks: - yield f"data: {chunk}" - yield "data: [DONE]" + yield f"data: {chunk}\n\n" + yield "data: [DONE]\n\n" mock_resp = MagicMock() - mock_resp.iter_lines.return_value = _iter_lines() + mock_resp.iter_text.return_value = _iter_text() mock_resp.status_code = 200 mock_resp.headers = {"content-type": "text/event-stream"} @@ -420,13 +449,13 @@ def test_acompletion_cloudflare_stream(sync_mode, monkeypatch): mock_post.assert_called_once() else: - async def _aiter_lines(): + async def _aiter_text() -> AsyncIterator[str]: for chunk in raw_chunks: - yield f"data: {chunk}" - yield "data: [DONE]" + yield f"data: {chunk}\n\n" + yield "data: [DONE]\n\n" mock_resp = MagicMock() - mock_resp.aiter_lines.return_value = _aiter_lines() + mock_resp.aiter_text.return_value = _aiter_text() mock_resp.status_code = 200 mock_resp.headers = {"content-type": "text/event-stream"} diff --git a/tests/unit/llms/codestral/__init__.py b/tests/unit/llms/codestral/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/codestral/completion/__init__.py b/tests/unit/llms/codestral/completion/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/codestral/completion/test_codestral_completion_handler.py b/tests/unit/llms/codestral/completion/test_codestral_completion_handler.py new file mode 100644 index 00000000000..9ce50d8ceca --- /dev/null +++ b/tests/unit/llms/codestral/completion/test_codestral_completion_handler.py @@ -0,0 +1,50 @@ +import json +from typing import Final + +import httpx +import pytest +import respx + +import litellm + +_LINE_SEPARATED_TEXT: Final = "before\u2028between\u2029after" + + +def _fim_stream_body(text: str) -> bytes: + chunks: Final = ( + {"id": "c1", "object": "chat.completion.chunk", "created": 1, "model": "codestral-latest", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": text}, "finish_reason": None}]}, + {"id": "c1", "object": "chat.completion.chunk", "created": 1, "model": "codestral-latest", + "choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}]}, + ) + events: Final = "".join(f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n" for chunk in chunks) + return f"{events}data: [DONE]\n\n".encode() + + +async def _streamed_texts(sync_stream: bool) -> list[str | None]: + request: Final = { + "model": "text-completion-codestral/codestral-latest", + "prompt": "def f(", + "stream": True, + "api_key": "codestral-key", + "max_retries": 0, + } + if sync_stream: + return [chunk.choices[0].text for chunk in litellm.text_completion(**request)] + return [chunk.choices[0].text async for chunk in await litellm.atext_completion(**request)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_stream", [True, False]) +async def test_fim_stream_keeps_unicode_line_separators_inside_a_delta( + sync_stream: bool, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://codestral.mistral.ai/v1/fim/completions").mock( + return_value=httpx.Response(200, content=_fim_stream_body(_LINE_SEPARATED_TEXT)) + ) + + texts: Final = await _streamed_texts(sync_stream) + + assert texts == [_LINE_SEPARATED_TEXT, None] + assert json.loads(route.calls.last.request.content)["stream"] is True diff --git a/tests/unit/llms/custom_httpx/test_aiohttp_handler.py b/tests/unit/llms/custom_httpx/test_aiohttp_handler.py index 83c38b65beb..bb66a70da84 100644 --- a/tests/unit/llms/custom_httpx/test_aiohttp_handler.py +++ b/tests/unit/llms/custom_httpx/test_aiohttp_handler.py @@ -1,11 +1,17 @@ +import json +from typing import Final from unittest.mock import AsyncMock, Mock, patch import aiohttp +import httpx import pytest import litellm +from litellm.llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig from litellm.llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport +from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.types.utils import ModelResponseStream class TestBaseLLMAIOHTTPHandler: @@ -445,3 +451,34 @@ class TestBaseLLMAIOHTTPHandler: # Should use transport, not connector mock_transport.get_valid_client_session.assert_called_once() assert result is mock_session_from_transport + + +def test_make_sync_call_stream_keeps_unicode_line_separators_inside_a_delta() -> None: + text: Final = "before\u2028between\u2029after" + chunk: Final = { + "id": "c1", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-5.5", + "choices": [{"index": 0, "delta": {"content": text}, "finish_reason": None}], + } + body: Final = f"data: {json.dumps(chunk, ensure_ascii=False)}\n\ndata: [DONE]\n\n".encode() + transport: Final = httpx.MockTransport(lambda request: httpx.Response(200, stream=httpx.ByteStream(body))) + + completion_stream, _ = BaseLLMAIOHTTPHandler().make_sync_call( + provider_config=AiohttpOpenAIChatConfig(), + api_base="https://example.com/v1/chat/completions", + headers={}, + data={"model": "gpt-5.5", "messages": []}, + model="gpt-5.5", + messages=[], + logging_obj=Mock(), + litellm_params={}, + timeout=5.0, + client=HTTPHandler(client=httpx.Client(transport=transport)), + ) + + contents: Final = [ + chunk.choices[0].delta.content for chunk in completion_stream if isinstance(chunk, ModelResponseStream) + ] + assert contents == [text] diff --git a/tests/unit/llms/custom_httpx/test_sse_lines.py b/tests/unit/llms/custom_httpx/test_sse_lines.py new file mode 100644 index 00000000000..9bb03f6da98 --- /dev/null +++ b/tests/unit/llms/custom_httpx/test_sse_lines.py @@ -0,0 +1,64 @@ +from collections.abc import AsyncIterator, Sequence +from typing import Final + +import pytest + +from litellm.llms.custom_httpx.sse_lines import ( + aiter_sse_lines, + iter_sse_lines, + split_sse_lines, + split_sse_lines_keepends, +) + +CASES: Final = ( + pytest.param(("data: a\u2028b\r\n\r\n",), ("data: a\u2028b", ""), id="u2028-stays-inside-the-line"), + pytest.param( + ("data: \u0085\u2029\x0b\x0c\x1c\x1d\x1e\n",), + ("data: \u0085\u2029\x0b\x0c\x1c\x1d\x1e",), + id="no-splitlines-only-separator-breaks-a-line", + ), + pytest.param(("a\r\nb\nc\rd\n",), ("a", "b", "c", "d"), id="crlf-lf-and-cr-each-end-a-line"), + pytest.param(("a\r", "\nb\n"), ("a", "b"), id="crlf-split-across-chunks-is-one-break"), + pytest.param(("a\r", "\n\r\n"), ("a", ""), id="crlf-split-then-blank-line"), + pytest.param(("a\r", "", "\nb\n"), ("a", "b"), id="empty-chunk-between-cr-and-lf"), + pytest.param(("a\r", "\r"), ("a", ""), id="cr-then-cr-is-two-breaks"), + pytest.param(("a\r\r\nb\n",), ("a", "", "b"), id="cr-then-crlf-in-one-chunk-is-two-breaks"), + pytest.param(("a\r\nb\r\n\r\n",), ("a", "b", ""), id="crlf-only-chunk"), + pytest.param(("da", "ta: x", "y\n"), ("data: xy",), id="line-spanning-chunks-is-joined"), + pytest.param(("data: x",), ("data: x",), id="unterminated-last-line-is-kept"), + pytest.param(("",), (), id="empty-body"), +) + + +async def _async_chunks(chunks: Sequence[str]) -> AsyncIterator[str]: + for chunk in chunks: + yield chunk + + +@pytest.mark.parametrize(("chunks", "expected"), CASES) +def test_iter_sse_lines_splits_only_on_cr_lf_and_crlf(chunks: Sequence[str], expected: Sequence[str]) -> None: + assert tuple(iter_sse_lines(chunks)) == tuple(expected) + + +@pytest.mark.parametrize(("chunks", "expected"), CASES) +async def test_aiter_sse_lines_splits_only_on_cr_lf_and_crlf(chunks: Sequence[str], expected: Sequence[str]) -> None: + assert tuple([line async for line in aiter_sse_lines(_async_chunks(chunks))]) == tuple(expected) + + +@pytest.mark.parametrize(("chunks", "expected"), CASES) +def test_split_sse_lines_splits_a_whole_body_only_on_cr_lf_and_crlf( + chunks: Sequence[str], expected: Sequence[str] +) -> None: + assert split_sse_lines("".join(chunks)) == tuple(expected) + + +@pytest.mark.parametrize( + ("text", "expected"), + ( + pytest.param("a\r\nb\rc\nd", ("a\r\n", "b\r", "c\n", "d"), id="each-break-stays-on-its-line"), + pytest.param("data: a\u2028b\u2029c\n\n", ("data: a\u2028b\u2029c\n", "\n"), id="u2028-and-u2029-stay-inside"), + pytest.param("", (), id="empty-body"), + ), +) +def test_split_sse_lines_keepends_splits_only_on_cr_lf_and_crlf(text: str, expected: Sequence[str]) -> None: + assert split_sse_lines_keepends(text) == tuple(expected) diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index b3a224209e9..eb2e1d8e347 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -1,6 +1,6 @@ import copy import json -from collections.abc import Iterator +from collections.abc import AsyncIterator, Iterator from typing import Final from unittest.mock import MagicMock, patch @@ -2160,13 +2160,13 @@ def mock_chat_response_nonanthropic_prompt_caching() -> Dict[str, Any]: def mock_http_handler_chat_streaming_response() -> MagicMock: mock_stream_chunks = mock_chat_streaming_response_chunks() - def mock_iter_lines(): + def mock_iter_text() -> Iterator[str]: for chunk in mock_stream_chunks: for line in chunk.splitlines(): - yield line + yield f"{line}\n" mock_response = MagicMock() - mock_response.iter_lines.side_effect = mock_iter_lines + mock_response.iter_text.side_effect = mock_iter_text mock_response.status_code = 200 return mock_response @@ -2175,13 +2175,13 @@ def mock_http_handler_chat_streaming_response() -> MagicMock: def mock_http_handler_chat_async_streaming_response() -> MagicMock: mock_stream_chunks = mock_chat_streaming_response_chunks() - async def mock_iter_lines(): + async def mock_aiter_text() -> AsyncIterator[str]: for chunk in mock_stream_chunks: for line in chunk.splitlines(): - yield line + yield f"{line}\n" mock_response = MagicMock() - mock_response.aiter_lines.return_value = mock_iter_lines() + mock_response.aiter_text.return_value = mock_aiter_text() mock_response.status_code = 200 return mock_response diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 04b791cf289..61e5545b8bf 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1329,7 +1329,7 @@ def test_streaming_surfaces_fireworks_response_fields(): raw_response = MagicMock() raw_response.status_code = 200 raw_response.headers = {} - raw_response.iter_lines = lambda: iter(sse_lines) + raw_response.iter_text = lambda: iter(f"{line}\n" for line in sse_lines) client = HTTPHandler() with patch.object(client, "post", return_value=raw_response): @@ -1788,7 +1788,7 @@ def test_streaming_preserves_selected_model_for_private_accounting(): raw_response = MagicMock() raw_response.status_code = 200 raw_response.headers = {} - raw_response.iter_lines = lambda: iter(sse_lines) + raw_response.iter_text = lambda: iter(f"{line}\n" for line in sse_lines) client = HTTPHandler() with patch.object(client, "post", return_value=raw_response): @@ -2275,7 +2275,7 @@ def test_completion_forwards_litellm_user_id_as_user(call_kwargs: dict[str, obje def test_completion_forwards_litellm_user_id_when_streaming() -> None: client: Final = _fireworks_chat_client() - client.post.return_value.iter_lines = lambda: iter(()) + client.post.return_value.iter_text = lambda: iter(()) litellm.completion( model="fireworks_ai/accounts/fireworks/models/kimi-k3", messages=[{"role": "user", "content": "hi"}], diff --git a/tests/unit/llms/gemini/google_genai/guardrail_translation/test_google_genai_guardrail_handler.py b/tests/unit/llms/gemini/google_genai/guardrail_translation/test_google_genai_guardrail_handler.py index 4119ce99423..53ddb1135b4 100644 --- a/tests/unit/llms/gemini/google_genai/guardrail_translation/test_google_genai_guardrail_handler.py +++ b/tests/unit/llms/gemini/google_genai/guardrail_translation/test_google_genai_guardrail_handler.py @@ -4,6 +4,7 @@ Tests for the Google GenAI generateContent guardrail translation handler. import json from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -190,6 +191,21 @@ async def test_streaming_raw_sse_chunks_accumulate_text_across_split_frames(): assert guardrail.apply_guardrail.call_args.kwargs["inputs"]["texts"] == ["harmful answer"] +@pytest.mark.asyncio +async def test_streaming_raw_sse_text_holding_unicode_line_separators_reaches_the_guardrail_whole() -> None: + guardrail: Final = _mock_guardrail(["clean"]) + text: Final = "harmful\u2028answer\u2029here" + payload: Final = {"candidates": [{"content": {"parts": [{"text": text}]}}]} + chunk: Final = f"data: {json.dumps(payload, ensure_ascii=False)}\r\n\r\n" + + await GoogleGenAIGenerateContentHandler().process_output_streaming_response( + responses_so_far=[chunk], + guardrail_to_apply=guardrail, + ) + + assert guardrail.apply_guardrail.call_args.kwargs["inputs"]["texts"] == [text] + + @pytest.mark.asyncio async def test_streaming_blocking_guardrail_exception_propagates(): handler = GoogleGenAIGenerateContentHandler() diff --git a/tests/unit/llms/langgraph/chat/test_langgraph_sse_iterator.py b/tests/unit/llms/langgraph/chat/test_langgraph_sse_iterator.py new file mode 100644 index 00000000000..06485362162 --- /dev/null +++ b/tests/unit/llms/langgraph/chat/test_langgraph_sse_iterator.py @@ -0,0 +1,32 @@ +import json +from typing import Final + +import httpx +import pytest + +from litellm.llms.langgraph.chat.sse_iterator import LangGraphSSEStreamIterator + +_LINE_SEPARATED_TEXT: Final = "before\u2028between\u2029after" + + +def _messages_event_stream(text: str) -> httpx.Response: + event: Final = ["messages", [[{"type": "AIMessageChunk", "content": text}, {}]]] + return httpx.Response(200, stream=httpx.ByteStream(f"data: {json.dumps(event, ensure_ascii=False)}\n\n".encode())) + + +async def _streamed_contents(iterator: LangGraphSSEStreamIterator, sync_stream: bool) -> list[str | None]: + if sync_stream: + return [chunk.choices[0].delta.content for chunk in iterator] + return [chunk.choices[0].delta.content async for chunk in iterator] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_stream", [True, False]) +async def test_stream_keeps_unicode_line_separators_inside_a_message_chunk(sync_stream: bool) -> None: + iterator: Final = LangGraphSSEStreamIterator( + response=_messages_event_stream(_LINE_SEPARATED_TEXT), model="langgraph/agent" + ) + + contents: Final = await _streamed_contents(iterator, sync_stream) + + assert contents == [_LINE_SEPARATED_TEXT, None] diff --git a/tests/unit/llms/oobabooga/chat/test_oobabooga.py b/tests/unit/llms/oobabooga/chat/test_oobabooga.py index 395a4fb5715..332b6913890 100644 --- a/tests/unit/llms/oobabooga/chat/test_oobabooga.py +++ b/tests/unit/llms/oobabooga/chat/test_oobabooga.py @@ -1,7 +1,12 @@ +import json +from typing import Final from unittest.mock import MagicMock, patch +import httpx +import respx import litellm +from litellm.llms.oobabooga.chat import oobabooga MOCK_COMPLETION_RESPONSE = { "choices": [{"message": {"role": "assistant", "content": "hi there"}}], @@ -50,3 +55,27 @@ def test_url_valued_model_still_targets_that_url(): mock_post.assert_called_once() called_url = mock_post.call_args[0][0] assert called_url == "https://sdk-user.example/v1/chat/completions" + + +def test_stream_keeps_unicode_line_separators_inside_a_data_line(respx_mock: respx.MockRouter) -> None: + text: Final = "before\u2028between\u2029after" + chunk: Final = {"choices": [{"index": 0, "delta": {"content": text}, "finish_reason": None}]} + data_line: Final = f"data: {json.dumps(chunk, ensure_ascii=False)}" + respx_mock.post("https://ooba.example/v1/chat/completions").mock( + return_value=httpx.Response(200, content=f"{data_line}\n\ndata: [DONE]\n\n".encode()) + ) + + lines: Final = oobabooga.completion( + model="my-model", + messages=[{"role": "user", "content": "hello"}], + api_base="https://ooba.example", + model_response=litellm.ModelResponse(), + print_verbose=print, + encoding=None, + api_key=None, + logging_obj=MagicMock(), + optional_params={"stream": True}, + litellm_params={}, + ) + + assert [line for line in lines if line] == [data_line, "data: [DONE]"] diff --git a/tests/unit/llms/openai_like/chat/test_openai_like_chat_handler.py b/tests/unit/llms/openai_like/chat/test_openai_like_chat_handler.py index 75b60b2dd6c..3df1efe311b 100644 --- a/tests/unit/llms/openai_like/chat/test_openai_like_chat_handler.py +++ b/tests/unit/llms/openai_like/chat/test_openai_like_chat_handler.py @@ -4,9 +4,10 @@ from typing import Final from unittest.mock import Mock import httpx +import pytest -from litellm.llms.custom_httpx.http_handler import HTTPHandler -from litellm.llms.openai_like.chat.handler import make_sync_call +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.openai_like.chat.handler import make_call, make_sync_call _FAKE_STREAMED_REPLY: Final = { "id": "chatcmpl-fake-stream", @@ -73,3 +74,35 @@ def test_fake_stream_keeps_reasoning_every_tool_call_and_token_detail(): assert [call.function.name for call in delta.tool_calls] == ["get_weather", "get_time"] assert chunks[0].usage.prompt_tokens_details.cached_tokens == 64 assert chunks[0].usage.completion_tokens_details.reasoning_tokens == 1920 + + +_LINE_SEPARATED_TEXT: Final = "before\u2028between\u2029after" + + +def _streamed_reply(request: httpx.Request) -> httpx.Response: + chunk: Final = {"id": "chatcmpl-separators", "choices": [{"index": 0, "delta": {"content": _LINE_SEPARATED_TEXT}}]} + body: Final = f"data: {json.dumps(chunk, ensure_ascii=False)}\n\ndata: [DONE]\n\n" + return httpx.Response(200, stream=httpx.ByteStream(body.encode())) + + +async def _streamed_chunks(sync_stream: bool) -> list[dict[str, object]]: + call_args: Final = { + "api_base": "https://openai-like.example/v1/chat/completions", + "headers": {}, + "data": json.dumps({"model": "openai/gpt-oss-120b", "messages": [], "stream": True}), + "model": "openai/gpt-oss-120b", + "messages": [], + "logging_obj": Mock(), + } + transport: Final = httpx.MockTransport(_streamed_reply) + if sync_stream: + return list(make_sync_call(client=HTTPHandler(client=httpx.Client(transport=transport)), **call_args)) + return [chunk async for chunk in await make_call(client=AsyncHTTPHandler(transport=transport), **call_args)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_stream", [True, False]) +async def test_stream_keeps_unicode_line_separators_inside_the_delta(sync_stream: bool) -> None: + chunks: Final = await _streamed_chunks(sync_stream) + + assert "".join(str(chunk["text"]) for chunk in chunks) == _LINE_SEPARATED_TEXT diff --git a/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py b/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py index f9d12b7eeea..36c319d9524 100644 --- a/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py +++ b/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py @@ -902,20 +902,14 @@ class TestSnowflakeChatCompletion: @pytest.mark.parametrize("sync_mode", [True, False]) def test_chat_completion_snowflake_stream(self, sync_mode): - raw_chunks = _streaming_chunks() + sse_body = "".join(f"data: {chunk}\n\n" for chunk in _streaming_chunks()) + "data: [DONE]\n\n" + mock_resp = httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=httpx.ByteStream(sse_body.encode()), + ) if sync_mode: - - def _iter_lines(): - for chunk in raw_chunks: - yield f"data: {chunk}" - yield "data: [DONE]" - - mock_resp = MagicMock() - mock_resp.iter_lines.return_value = _iter_lines() - mock_resp.status_code = 200 - mock_resp.headers = {"content-type": "text/event-stream"} - with patch.object(HTTPHandler, "post", return_value=mock_resp) as mock_post: response = completion( model="snowflake/mistral-7b", @@ -930,16 +924,6 @@ class TestSnowflakeChatCompletion: mock_post.assert_called_once() else: - async def _aiter_lines(): - for chunk in raw_chunks: - yield f"data: {chunk}" - yield "data: [DONE]" - - mock_resp = MagicMock() - mock_resp.aiter_lines.return_value = _aiter_lines() - mock_resp.status_code = 200 - mock_resp.headers = {"content-type": "text/event-stream"} - async def _run(): with patch.object( AsyncHTTPHandler, @@ -964,8 +948,8 @@ class TestSnowflakeChatCompletion: chunks_received = asyncio.run(_run()) - assert len(chunks_received) > 0 content = "".join(c.choices[0].delta.content for c in chunks_received if c.choices[0].delta.content) + assert content == "The sky is blue" async def test_snowflake_claude_async_completion_inlines_remote_images_off_the_event_loop(async_only_image_fetch): diff --git a/tests/unit/llms/test_predibase_transformation.py b/tests/unit/llms/test_predibase_transformation.py index a02840ed4ef..7afbd05b221 100644 --- a/tests/unit/llms/test_predibase_transformation.py +++ b/tests/unit/llms/test_predibase_transformation.py @@ -1,3 +1,4 @@ +import json from typing import Final from unittest.mock import AsyncMock, Mock @@ -639,3 +640,49 @@ def test_predibase_completion_passes_existing_config_to_async_completion(monkeyp assert result == "async-result" assert captured["async_kwargs"]["predibase_config"] is captured["config_instance"] + + +_LINE_SEPARATED_TEXT: Final = "before\u2028between\u2029after" + + +def _generate_stream_body(text: str) -> bytes: + events: Final = ( + {"token": {"text": text}}, + {"token": {"text": ""}, "generated_text": text, "details": {"finish_reason": "eos_token"}}, + ) + return "".join(f"data:{json.dumps(event, ensure_ascii=False)}\n\n" for event in events).encode() + + +async def _streamed_contents(sync_stream: bool) -> list[str | None]: + request: Final = { + "model": "predibase/llama-3-8b-instruct", + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "predibase_tenant_id": "tenant-1", + "api_key": "pb-key", + "max_retries": 0, + } + if sync_stream: + return [chunk.choices[0].delta.content for chunk in litellm.completion(**request)] + return [chunk.choices[0].delta.content async for chunk in await litellm.acompletion(**request)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_stream", [True, False]) +async def test_predibase_stream_keeps_unicode_line_separators_inside_a_token( + sync_stream: bool, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_hf_tokenizer_download", True) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr( + litellm, + "custom_prompt_dict", + {"llama-3-8b-instruct": {"roles": {}, "initial_prompt_value": "", "final_prompt_value": ""}}, + ) + respx_mock.post( + "https://serving.app.predibase.com/tenant-1/deployments/v2/llms/llama-3-8b-instruct/generate_stream" + ).mock(return_value=httpx.Response(200, content=_generate_stream_body(_LINE_SEPARATED_TEXT))) + + contents: Final = await _streamed_contents(sync_stream) + + assert contents == [_LINE_SEPARATED_TEXT, None] diff --git a/tests/unit/llms/triton/test_triton.py b/tests/unit/llms/triton/test_triton.py index 7c0de603a3e..238c30816b2 100644 --- a/tests/unit/llms/triton/test_triton.py +++ b/tests/unit/llms/triton/test_triton.py @@ -1,5 +1,7 @@ import json import traceback +from collections.abc import Iterator +from typing import Final from unittest.mock import MagicMock, patch import httpx @@ -210,8 +212,8 @@ def test_completion_triton_generate_api(stream, monkeypatch, respx_mock): mock_response = MagicMock() if stream: - def mock_iter_lines(): - mock_output = "".join( + def mock_iter_text() -> Iterator[str]: + mock_output: Final = "".join( [ 'data: {"model_name":"ensemble","model_version":"1","sequence_end":false,"sequence_id":0,"sequence_start":false,"text_output":"' + t @@ -219,10 +221,9 @@ def test_completion_triton_generate_api(stream, monkeypatch, respx_mock): for t in ["I", " am", " an", " AI", " assistant"] ] ) - for out in mock_output.split("\n"): - yield out + yield mock_output - mock_response.iter_lines = mock_iter_lines + mock_response.iter_text = mock_iter_text else: def return_val(): diff --git a/tests/unit/llms/vertex_ai/agent_engine/test_transformation.py b/tests/unit/llms/vertex_ai/agent_engine/test_transformation.py index 06aaca44e8a..20333fd0465 100644 --- a/tests/unit/llms/vertex_ai/agent_engine/test_transformation.py +++ b/tests/unit/llms/vertex_ai/agent_engine/test_transformation.py @@ -14,7 +14,7 @@ import pytest from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, get_async_httpx_client from litellm.llms.vertex_ai.agent_engine.sse_iterator import ( VertexAgentEngineResponseIterator, ) @@ -170,3 +170,48 @@ async def test_async_stream_wrapper_without_a_client_posts_through_the_cached_ve assert type(wrapper) is CustomStreamWrapper assert [str(request.url) for request in sent_requests] == [api_base] assert json.loads(sent_requests[0].content) == {"class_method": "stream_query"} + + +_LINE_SEPARATED_TEXT: Final = "before\u2028between\u2029after" + + +def _separated_text_reply(request: httpx.Request) -> httpx.Response: + chunk: Final = {"content": {"parts": [{"text": _LINE_SEPARATED_TEXT}], "role": "model"}, "finish_reason": "STOP"} + return httpx.Response(200, stream=httpx.ByteStream(f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n".encode())) + + +async def _streamed_text(sync_stream: bool) -> str: + messages: Final = [{"role": "user", "content": "hi"}] + call_args: Final = { + "model": "agent_engine/123", + "custom_llm_provider": "vertex_ai", + "logging_obj": Logging( + model="agent_engine/123", + messages=messages, + stream=True, + call_type="completion" if sync_stream else "acompletion", + start_time=datetime(2026, 1, 1), + litellm_call_id="call-separators", + function_id="fn-separators", + ), + "api_base": "https://us-central1-aiplatform.googleapis.com/v1/reasoningEngines/123:streamQuery", + "headers": {}, + "data": {"class_method": "stream_query"}, + "messages": messages, + "litellm_params": {}, + } + transport: Final = httpx.MockTransport(_separated_text_reply) + if sync_stream: + sync_wrapper: Final = VertexAgentEngineConfig().get_sync_custom_stream_wrapper( + client=HTTPHandler(client=httpx.Client(transport=transport)), **call_args + ) + return "".join(chunk.choices[0].delta.content or "" for chunk in sync_wrapper) + async_wrapper: Final = await VertexAgentEngineConfig().get_async_custom_stream_wrapper( + client=AsyncHTTPHandler(transport=transport), **call_args + ) + return "".join([chunk.choices[0].delta.content or "" async for chunk in async_wrapper]) + + +@pytest.mark.parametrize("sync_stream", [True, False]) +async def test_stream_keeps_unicode_line_separators_inside_the_text(sync_stream: bool) -> None: + assert await _streamed_text(sync_stream) == _LINE_SEPARATED_TEXT diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 3e1bca18f08..67c78b62151 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -3,8 +3,9 @@ import json import logging import re import threading -from collections.abc import Awaitable, Callable, Iterator, Mapping +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Mapping from copy import deepcopy +from itertools import pairwise from typing import Final, List, Literal, cast, get_args from unittest.mock import MagicMock, Mock, patch @@ -34,7 +35,7 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( ) from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.vertex_ai import GeminiFinishReason, RequestBody, UsageMetadata -from litellm.types.utils import ChoiceLogprobs, ModelInfo, Usage +from litellm.types.utils import ChoiceLogprobs, ModelInfo, ModelResponseStream, Usage from litellm.utils import _invalidate_model_cost_lowercase_map, CustomStreamWrapper from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.integrations.custom_logger import CustomLogger @@ -10125,7 +10126,7 @@ def test_vertex_ai_streaming_response_id(): mock_response = MagicMock() mock_response.status_code = 200 - mock_response.iter_lines = MagicMock(return_value=stream_response()) + mock_response.iter_text = MagicMock(return_value=stream_response()) return mock_response logging_obj = MagicMock() @@ -10690,6 +10691,63 @@ async def test_gemini_sends_a_boolean_for_a_gateway_default_written_as_text_or_n assert sent["includeThoughts"] is expected +class _BodyInChunks(httpx.SyncByteStream, httpx.AsyncByteStream): + def __init__(self, chunks: tuple[bytes, ...]) -> None: + self._chunks: Final = chunks + + def __iter__(self) -> Iterator[bytes]: + yield from self._chunks + + async def __aiter__(self) -> AsyncIterator[bytes]: + for chunk in self._chunks: + yield chunk + + +def _split_at(body: bytes, cuts: tuple[int, ...]) -> tuple[bytes, ...]: + return tuple(body[start:end] for start, end in pairwise((0, *sorted(cuts), len(body)))) + + +async def _gemini_stream(is_async: bool, chunks: tuple[bytes, ...]) -> list[ModelResponseStream]: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, stream=_BodyInChunks(chunks), headers={"content-type": "text/event-stream"}) + + kwargs: Final = { + "model": "gemini/gemini-3.8-flash", + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "stream_options": {"include_usage": True}, + "api_key": "fake-gemini-key", + } + if is_async: + stream: Final = await litellm.acompletion( + **kwargs, client=AsyncHTTPHandler(transport=httpx.MockTransport(handler)) + ) + return [chunk async for chunk in stream] + return list( + litellm.completion(**kwargs, client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler)))) + ) + + +@pytest.mark.parametrize("is_async", [True, False]) +async def test_gemini_stream_keeps_text_and_usage_when_a_part_contains_a_unicode_line_separator(is_async: bool) -> None: + events: Final = ( + {"candidates": [{"content": {"role": "model", "parts": [{"text": "line one\u2028line two"}]}}]}, + { + "candidates": [{"content": {"role": "model", "parts": [{"text": " and three"}]}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 8, "totalTokenCount": 18}, + }, + ) + body: Final = "".join(f"data: {json.dumps(event, ensure_ascii=False)}\r\n\r\n" for event in events).encode() + cut_inside_separator_and_crlf: Final = (body.index(b"\xe2\x80\xa8") + 1, body.index(b"\r") + 1) + + received: Final = await _gemini_stream(is_async, _split_at(body, cut_inside_separator_and_crlf)) + + usage: Final = next(chunk.usage for chunk in reversed(received) if getattr(chunk, "usage", None)) + text: Final = "".join(c.choices[0].delta.content or "" for c in received if c.choices) + assert text == "line one\u2028line two and three" + assert (usage.prompt_tokens, usage.completion_tokens) == (10, 8) + + @pytest.mark.parametrize("chunk_indices", [[0, 1], [2, 1, 0]]) def test_grounding_support_preserves_every_web_source(chunk_indices: list[int]) -> None: metadata: Final = { diff --git a/tests/unit/llms/vertex_ai/test_http_status_201.py b/tests/unit/llms/vertex_ai/test_http_status_201.py index 34502d5cebe..0ed469acfa7 100644 --- a/tests/unit/llms/vertex_ai/test_http_status_201.py +++ b/tests/unit/llms/vertex_ai/test_http_status_201.py @@ -39,8 +39,8 @@ class TestVertexAIHTTPStatus201(unittest.TestCase): # Create a mock response with status code 201 mock_response = MagicMock() mock_response.status_code = 201 - mock_response.aiter_lines = MagicMock() - mock_response.aiter_lines.return_value = ["test response"] + mock_response.aiter_text = MagicMock() + mock_response.aiter_text.return_value = ["test response"] # Setup mock client mock_client = MagicMock() @@ -76,8 +76,8 @@ class TestVertexAIHTTPStatus201(unittest.TestCase): # Create a mock response with status code 201 mock_response = MagicMock() mock_response.status_code = 201 - mock_response.iter_lines = MagicMock() - mock_response.iter_lines.return_value = ["test response"] + mock_response.iter_text = MagicMock() + mock_response.iter_text.return_value = ["test response"] mock_post.return_value = mock_response # Call the make_sync_call function diff --git a/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py b/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py index a7d7ac5627c..7abe6ad7799 100644 --- a/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py +++ b/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py @@ -4,10 +4,11 @@ Mock tests for A2A endpoints. Tests that invoke_agent_a2a properly integrates with add_litellm_data_to_request. """ +import asyncio import json import socket import sys -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import AsyncIterator, Awaitable, Callable, Mapping from contextlib import AbstractContextManager, ExitStack from dataclasses import dataclass from typing import Final @@ -24,6 +25,21 @@ from litellm.types.agents import AgentCaller AddLiteLLMData = Callable[..., Awaitable[dict[str, object]]] +class _SSELineStream(httpx.AsyncByteStream): + def __init__(self, lines: tuple[str, ...], first_line_delay: float) -> None: + self._lines = lines + self._first_line_delay = first_line_delay + + async def __aiter__(self) -> AsyncIterator[bytes]: + await asyncio.sleep(self._first_line_delay) + for line in self._lines: + yield f"{line}\n".encode() + + +def _sse_upstream_response(*lines: str, first_line_delay: float = 0.0) -> httpx.Response: + return httpx.Response(200, stream=_SSELineStream(lines, first_line_delay)) + + @dataclass(frozen=True, slots=True) class CapturedAgentCall: request_id: object @@ -941,13 +957,7 @@ async def test_task_methods_extract_litellm_params_before_forwarding(method: str mock_http_response.json.return_value = upstream_response mock_http_response.is_success = True - async def fake_aiter_lines(): - yield 'data: {"jsonrpc":"2.0","id":"req-1","result":{"taskId":"task-1"}}' - - mock_resp = AsyncMock() - mock_resp.is_success = True - mock_resp.aiter_lines = fake_aiter_lines - mock_resp.aclose = AsyncMock() + mock_resp = _sse_upstream_response('data: {"jsonrpc":"2.0","id":"req-1","result":{"taskId":"task-1"}}') mock_async_client = MagicMock() mock_async_client.build_request = MagicMock(return_value=MagicMock()) @@ -982,8 +992,8 @@ async def test_task_methods_extract_litellm_params_before_forwarding(method: str user_api_key_dict=user_api_key_dict, ) if method == "tasks/resubscribe": - async for _ in response.body_iterator: - pass + streamed = "".join([chunk async for chunk in response.body_iterator]) + assert "task-1" in streamed if method == "tasks/resubscribe": forwarded_body = mock_async_client.build_request.call_args.kwargs["json"] @@ -1007,14 +1017,7 @@ async def test_subscribe_to_task_returns_sse_stream(): 'data: {"jsonrpc":"2.0","id":"req-1","result":{"taskId":"task-1","status":{"state":"completed"}}}', ] - async def fake_aiter_lines(): - for line in sse_lines: - yield line - - mock_resp = AsyncMock() - mock_resp.is_success = True - mock_resp.aiter_lines = fake_aiter_lines - mock_resp.aclose = AsyncMock() + mock_resp = _sse_upstream_response(*sse_lines) mock_async_client = MagicMock() mock_async_client.build_request = MagicMock(return_value=MagicMock()) @@ -1064,13 +1067,9 @@ async def test_subscribe_to_task_calls_pre_call_hook(): mock_request = _make_request_mock("tasks/resubscribe", {"id": "task-1"}) user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") - async def fake_aiter_lines(): - yield 'data: {"jsonrpc":"2.0","id":"req-1","result":{"taskId":"task-1","status":{"state":"completed"}}}' - - mock_resp = AsyncMock() - mock_resp.is_success = True - mock_resp.aiter_lines = fake_aiter_lines - mock_resp.aclose = AsyncMock() + mock_resp = _sse_upstream_response( + 'data: {"jsonrpc":"2.0","id":"req-1","result":{"taskId":"task-1","status":{"state":"completed"}}}' + ) mock_async_client = MagicMock() mock_async_client.build_request = MagicMock(return_value=MagicMock()) @@ -1117,9 +1116,9 @@ async def test_subscribe_to_task_calls_pre_call_hook(): ) assert response.media_type == "text/event-stream" - async for _ in response.body_iterator: - pass + streamed = "".join([chunk async for chunk in response.body_iterator]) + assert "completed" in streamed mock_proxy_logging.pre_call_hook.assert_awaited_once() call_kwargs = mock_proxy_logging.pre_call_hook.await_args.kwargs assert call_kwargs.get("call_type") == "asend_message" @@ -1149,16 +1148,10 @@ async def test_subscribe_to_task_runs_post_call_streaming_guardrail(): mock_request = _make_request_mock("tasks/resubscribe", {"id": "task-1"}) user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") - async def fake_aiter_lines(): - yield ( - 'data: {"jsonrpc":"2.0","id":"req-1","result":' - '{"kind":"message","parts":[{"kind":"text","text":"resubscribe-secret"}]}}' - ) - - mock_resp = AsyncMock() - mock_resp.is_success = True - mock_resp.aiter_lines = fake_aiter_lines - mock_resp.aclose = AsyncMock() + mock_resp = _sse_upstream_response( + 'data: {"jsonrpc":"2.0","id":"req-1","result":' + '{"kind":"message","parts":[{"kind":"text","text":"resubscribe-secret"}]}}' + ) mock_async_client = MagicMock() mock_async_client.build_request = MagicMock(return_value=MagicMock()) @@ -1600,14 +1593,7 @@ async def test_pascal_method_names_normalize_to_wire_format(pascal_method: str, mock_http_response.is_success = True mock_http_response.raise_for_status = MagicMock() - async def _empty_aiter_lines(): - return - yield # make it an async generator - - mock_sse_resp = AsyncMock() - mock_sse_resp.is_success = True - mock_sse_resp.aiter_lines = _empty_aiter_lines - mock_sse_resp.aclose = AsyncMock() + mock_sse_resp = _sse_upstream_response() mock_async_client = MagicMock() mock_async_client.build_request = MagicMock(return_value=MagicMock()) @@ -2349,13 +2335,7 @@ async def test_task_methods_forward_trace_header(method: str): mock_http_response.json.return_value = upstream_response mock_http_response.is_success = True - async def fake_aiter_lines(): - yield 'data: {"jsonrpc":"2.0","id":"req-1","result":{"taskId":"task-1"}}' - - mock_resp = AsyncMock() - mock_resp.is_success = True - mock_resp.aiter_lines = fake_aiter_lines - mock_resp.aclose = AsyncMock() + mock_resp = _sse_upstream_response('data: {"jsonrpc":"2.0","id":"req-1","result":{"taskId":"task-1"}}') mock_async_client = MagicMock() mock_async_client.build_request = MagicMock(return_value=MagicMock()) @@ -2390,8 +2370,8 @@ async def test_task_methods_forward_trace_header(method: str): user_api_key_dict=user_api_key_dict, ) if method == "tasks/resubscribe": - async for _ in response.body_iterator: - pass + streamed = "".join([chunk async for chunk in response.body_iterator]) + assert "task-1" in streamed if method == "tasks/resubscribe": forwarded_headers = mock_async_client.build_request.call_args.kwargs["headers"] @@ -2646,17 +2626,12 @@ def test_served_version_falls_back_to_header_when_unconfigured(): assert _served_version(_agent(None), _request_with_a2a_header(None)) == "0.3" -def _sse_agent_handler(lines): - mock_resp = AsyncMock() - mock_resp.is_success = True - mock_resp.aiter_lines = lines - mock_resp.aclose = AsyncMock() - - mock_async_client = MagicMock() +def _sse_agent_handler(upstream_response: httpx.Response) -> MagicMock: + mock_async_client: Final = MagicMock() mock_async_client.build_request = MagicMock(return_value=MagicMock()) - mock_async_client.send = AsyncMock(return_value=mock_resp) + mock_async_client.send = AsyncMock(return_value=upstream_response) - mock_handler = MagicMock() + mock_handler: Final = MagicMock() mock_handler.client = mock_async_client return mock_handler @@ -2685,13 +2660,13 @@ async def test_forward_jsonrpc_sse_pings_while_the_upstream_agent_is_still_silen monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", 0.05) - async def _slow_lines(): - await asyncio.sleep(0.3) - yield 'data: {"jsonrpc": "2.0", "id": "req-1", "result": {"kind": "task"}}' + slow_upstream = _sse_upstream_response( + 'data: {"jsonrpc": "2.0", "id": "req-1", "result": {"kind": "task"}}', first_line_delay=0.3 + ) with patch( "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", - return_value=_sse_agent_handler(_slow_lines), + return_value=_sse_agent_handler(slow_upstream), ): response = await _resubscribe_response() assert response.headers["x-accel-buffering"] == "no" @@ -2713,12 +2688,11 @@ async def test_forward_jsonrpc_sse_is_untouched_while_keepalives_are_unconfigure monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", None) - async def _lines(): - yield 'data: {"jsonrpc": "2.0", "id": "req-1", "result": {"kind": "task"}}' - with patch( "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", - return_value=_sse_agent_handler(_lines), + return_value=_sse_agent_handler( + _sse_upstream_response('data: {"jsonrpc": "2.0", "id": "req-1", "result": {"kind": "task"}}') + ), ): response = await _resubscribe_response() assert "x-accel-buffering" not in response.headers @@ -2728,6 +2702,28 @@ async def test_forward_jsonrpc_sse_is_untouched_while_keepalives_are_unconfigure assert json.loads(chunks[-1].removeprefix("data: "))["result"]["kind"] == "task" +@pytest.mark.asyncio +async def test_forward_jsonrpc_sse_keeps_unicode_line_separators_inside_an_event(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", None) + separated_text: Final = "before\u2028between\u2029after" + event: Final = { + "jsonrpc": "2.0", + "id": "req-1", + "result": {"kind": "message", "parts": [{"kind": "text", "text": separated_text}]}, + } + + with patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=_sse_agent_handler(_sse_upstream_response(f"data: {json.dumps(event, ensure_ascii=False)}")), + ): + response: Final = await _resubscribe_response() + chunks: Final = [chunk async for chunk in response.body_iterator] + + assert [json.loads(chunk.removeprefix("data: "))["result"]["parts"][0]["text"] for chunk in chunks] == [ + separated_text + ] + + async def _stream_message_response(): from litellm.proxy.agent_endpoints.a2a_endpoints import _handle_stream_message diff --git a/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py b/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py index 85bb8304b95..5911fcdc073 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py +++ b/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py @@ -4,6 +4,7 @@ Tests for restamping the public model on Anthropic Messages streaming chunks. import json from collections.abc import AsyncIterator, Sequence +from typing import Final import pytest @@ -46,6 +47,28 @@ def test_restamps_sse_bytes_frame(): assert b"event: message_start" in restamped +def test_restamps_sse_bytes_frame_holding_a_line_separator() -> None: + payload: Final = { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5-20251001", + "content": [{"type": "text", "text": "alpha\u2028beta"}], + }, + } + frame: Final = f"event: message_start\ndata: {json.dumps(payload, ensure_ascii=False)}\n\n".encode() + + restamped: Final = restamp_anthropic_stream_chunk_model(frame, "claude-auto-1") + + assert isinstance(restamped, bytes) + data_line: Final = next(line for line in restamped.decode().split("\n") if line.startswith("data:")) + message: Final = json.loads(data_line[len("data:") :])["message"] + assert message["model"] == "claude-auto-1" + assert message["content"] == [{"type": "text", "text": "alpha\u2028beta"}] + + def test_restamps_event_dict(): chunk = {"type": "message_start", "message": {"id": "msg_1", "model": "claude-sonnet-4-6"}} diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index e09a179eadf..47b6af9aeb3 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -2503,6 +2503,33 @@ def test_stream_was_interrupted_skips_data_lines_that_are_not_json_objects( assert AnthropicPassthroughLoggingHandler._stream_was_interrupted(all_chunks) is interrupted + +def test_stream_was_interrupted_reads_a_message_delta_whose_json_holds_unicode_line_separators() -> None: + message_delta: Final = { + "type": "message_delta", + "delta": {"stop_reason": "stop_sequence", "stop_sequence": "END\u2028OF\u2029TURN"}, + "usage": {"output_tokens": 3}, + } + all_chunks: Final = [ + 'data: {"type": "content_block_stop", "index": 0}', + f"data: {json.dumps(message_delta, ensure_ascii=False)}", + ] + + assert AnthropicPassthroughLoggingHandler._stream_was_interrupted(all_chunks) is False + + +def test_extract_model_reads_a_message_start_whose_json_holds_unicode_line_separators() -> None: + message_start: Final = { + "type": "message_start", + "message": { + "model": "claude-sonnet-5", + "content": [{"type": "text", "text": "Prefill\u2028line\u2029two"}], + }, + } + chunk: Final = f"event: message_start\ndata: {json.dumps(message_start, ensure_ascii=False)}\n\n" + + assert AnthropicPassthroughLoggingHandler._extract_model_from_anthropic_chunks([chunk]) == "claude-sonnet-5" + @pytest.mark.asyncio async def test_anthropic_passthrough_handler(mock_httpx_response, mock_response, mock_logging_obj): """ @@ -3004,3 +3031,24 @@ def test_anthropic_passthrough_stream_bills_the_bedrock_guardrail_trace_of_the_m (entry,) = _recorded_guardrail_entries(mock_logging_obj) assert entry["guardrail_usage"] == {"topicPolicyUnits": 3, "contentPolicyUnits": 3} assert entry["guardrail_cost"] == pytest.approx(6 * 0.00015) + + +def test_anthropic_passthrough_stream_bills_a_bedrock_guardrail_trace_whose_json_holds_unicode_line_separators( + monkeypatch: pytest.MonkeyPatch, mock_logging_obj: LiteLLMLoggingObj, all_chunks: list[str] +) -> None: + monkeypatch.setattr(litellm, "model_cost", _BEDROCK_TRACE_PRICING) + mock_logging_obj.model_call_details["litellm_params"] = {"metadata": {}} + trace: Final = {"guardrail": {**_BEDROCK_INVOKE_TRACE["guardrail"], "actionReason": "Blocked\u2028by\u2029policy"}} + stop_event: Final = json.dumps({"type": "message_stop", "amazon-bedrock-trace": trace}, ensure_ascii=False) + AnthropicPassthroughLoggingHandler.handle_logging_anthropic_collected_chunks( + litellm_logging_obj=mock_logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + request_body={"model": "claude", "messages": []}, + endpoint_type=MagicMock(), + start_time=datetime(2025, 1, 1), + all_chunks=[*all_chunks, f"event: message_stop\ndata: {stop_event}\n\n"], + end_time=datetime(2025, 1, 1), + ) + (entry,) = _recorded_guardrail_entries(mock_logging_obj) + assert entry["guardrail_usage"] == {"topicPolicyUnits": 3, "contentPolicyUnits": 3} diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_tinyfish_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_tinyfish_passthrough_logging_handler.py index 1b83c5140ca..b0f84f2ecbe 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_tinyfish_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_tinyfish_passthrough_logging_handler.py @@ -1,6 +1,7 @@ import asyncio import json from datetime import datetime +from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx @@ -267,6 +268,12 @@ class TestRunIdFromSseFrames: frames = b'data: {"run_id": "run-7", "event": "INITIALIZED"}\n\ndata: {"run_id": "run-7", "event": "ACTION"}\n\n' assert run_id_from_sse_frames(frames) == "run-7" + def test_finds_run_id_in_a_frame_whose_json_holds_unicode_line_separators(self) -> None: + event: Final = {"run_id": "run-7", "event": "ACTION", "message": "Clicked\u2028Submit\u2029button"} + frames: Final = f"data: {json.dumps(event, ensure_ascii=False)}\n\n".encode() + + assert run_id_from_sse_frames(frames) == "run-7" + def test_skips_frames_without_run_id(self): frames = b': keepalive\n\ndata: not-json\n\ndata: {"event": "HEARTBEAT"}\n\n' assert run_id_from_sse_frames(frames) is None diff --git a/tests/unit/proxy/pass_through_endpoints/test_managed_id_rewriter.py b/tests/unit/proxy/pass_through_endpoints/test_managed_id_rewriter.py index 49bc2e9c29e..2496aa47d22 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_managed_id_rewriter.py +++ b/tests/unit/proxy/pass_through_endpoints/test_managed_id_rewriter.py @@ -30,7 +30,7 @@ from litellm.llms.base_llm.managed_resources.utils import( resolve_passthrough_managed_id_provider, ) from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome -from typing import Any +from typing import Any, Final def _user() -> UserAPIKeyAuth: @@ -254,6 +254,37 @@ async def test_streamed_response_with_cr_only_frame_delimiters_is_still_owned_an assert output == _response_stream_bytes(managed_id).replace(b"\n", b"\r") +def _instructed_response_stream_bytes(raw_id: str, instructions: str) -> bytes: + events: Final = ( + {"type": "response.created", "response": {"id": raw_id, "status": "in_progress", "instructions": instructions}}, + {"type": "response.completed", "response": {"id": raw_id, "status": "completed", "instructions": instructions}}, + ) + return b"".join( + f"event: {event['type']}\ndata: {json.dumps(event, ensure_ascii=False)}\n\n".encode() for event in events + ) + + +@pytest.mark.asyncio +async def test_streamed_response_whose_json_holds_unicode_line_separators_is_still_owned_and_rewritten() -> None: + instructions: Final = "Answer\u2028in\u2029verse" + pc: Final = _prisma_client() + + output: Final = await _collect( + rewrite_streamed_response_ids( + stream=_chunks(_instructed_response_stream_bytes(RAW_RESPONSE_ID, instructions), 4096), + provider="openai", + method="POST", + route="/openai_passthrough/v1/responses", + user_api_key_dict=_user(), + prisma_client=pc, + ) + ) + + pc.db.litellm_managedobjecttable.upsert.assert_awaited_once() + managed_id: Final = pc.db.litellm_managedobjecttable.upsert.await_args.kwargs["data"]["create"]["unified_object_id"] + assert output == _instructed_response_stream_bytes(managed_id, instructions) + + @pytest.mark.asyncio async def test_streamed_bytes_untouched_on_routes_without_a_response_id(): pc = _prisma_client() diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 63e38ce533a..42acad28488 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -17,7 +17,7 @@ following the OpenAI Response API format. import asyncio import json from datetime import datetime, timezone -from typing import Any, Dict, Optional +from typing import Any, Dict, Final, Optional from unittest.mock import AsyncMock, Mock, patch import pytest @@ -1400,7 +1400,7 @@ def _make_sse_stream(events: list) -> Mock: async def _body_iterator(): for event in events: - yield f"data: {json.dumps(event)}" + yield f"data: {json.dumps(event, ensure_ascii=False)}" yield "data: [DONE]" mock_response = Mock() @@ -1639,6 +1639,33 @@ class TestBackgroundStreamingTerminalEvents: assert final_call.kwargs["status"] == "completed" assert final_call.kwargs["usage"] == {"input_tokens": 10, "output_tokens": 50} + @pytest.mark.asyncio + async def test_response_completed_whose_json_holds_unicode_line_separators_keeps_its_output(self) -> None: + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + item: Final = { + "id": "item_1", + "type": "message", + "content": [{"type": "output_text", "text": "Line\u2028one\u2029two"}], + } + events: Final = [ + {"type": "response.completed", "response": {"id": "resp_123", "status": "completed", "output": [item]}} + ] + handler: Final = AsyncMock(spec=ResponsePollingHandler) + + with patch( + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock(return_value=_make_sse_stream(events)) + await background_streaming_task(**_make_background_streaming_kwargs("poll_u2028", handler)) + + calls: Final = handler.update_state.call_args_list + outputs: Final = [call.kwargs["output"] for call in calls if "output" in call.kwargs] + assert outputs[-1] == [item] + assert calls[-1].kwargs["status"] == "completed" + @pytest.mark.asyncio async def test_fallback_status_derived_from_event_type_when_status_field_missing( self, diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index ed90acbd2fa..0b971bc1696 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -1288,6 +1288,106 @@ async def test_reasoning_item_closes_before_a_tool_call_from_the_same_chunk_open ] +def _text_opening_a_tool_call_chunk(content: str, arguments: str = "") -> ModelResponseStream: + return ModelResponseStream( + id=CHAT_COMPLETION_ID, + created=1748575031, + model="claude-haiku-5-5", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + delta=Delta( + role="assistant", + content=content, + tool_calls=[ + { + "id": "call_weather", + "type": "function", + "function": {"name": "get_weather", "arguments": arguments}, + "index": 0, + } + ], + ), + finish_reason=None, + ) + ], + ) + + +def _tool_arguments_chunk(arguments: str, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id=CHAT_COMPLETION_ID, + created=1748575031, + model="claude-haiku-5-5", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + delta=Delta( + content=None, + tool_calls=[{"type": "function", "function": {"arguments": arguments}, "index": 0}], + ), + finish_reason=finish_reason, + ) + ], + ) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_tool_call_opened_in_a_text_chunk_streams_each_argument_delta(sync_mode: bool) -> None: + argument_pieces: Final = ('{"', "city", '"', ': "', "Paris", '"', "}") + iterator: Final = _build_iterator( + [ + _text_opening_a_tool_call_chunk(""), + *(_tool_arguments_chunk(piece) for piece in argument_pieces), + _chunk("", finish_reason="tool_calls"), + ] + ) + + events: Final = await _collect_events(iterator, sync_mode) + + assert _event_text(events, ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "" + assert [ + event.delta + for event in events + if getattr(event, "type", None) == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA + ] == list(argument_pieces) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_tool_call_sharing_a_chunk_with_text_opens_after_the_whole_message(sync_mode: bool) -> None: + iterator: Final = _build_iterator( + [ + _text_opening_a_tool_call_chunk("Checking ", arguments='{"city": "Paris"}'), + _chunk("the "), + _chunk("weather now."), + _chunk("", finish_reason="tool_calls"), + ] + ) + + events: Final = await _collect_events(iterator, sync_mode) + + event_types: Final = [getattr(event, "type", None) for event in events] + function_call_added_at: Final = next( + position + for position, event in enumerate(events) + if event_types[position] == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED and event.item.type == "function_call" + ) + streamed: Final = [ + event + for event in events + if getattr(event, "type", None) + in (ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA) + ] + assert ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA not in event_types[function_call_added_at:] + assert [event.sequence_number for event in streamed] == sorted(event.sequence_number for event in streamed) + assert _event_text(events, ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Checking the weather now." + assert _event_text(events, ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA) == '{"city": "Paris"}' + + @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_tool_only_stream_puts_the_function_call_at_output_index_zero(sync_mode: bool): diff --git a/tests/unit/sandbox/test_e2b_sandbox.py b/tests/unit/sandbox/test_e2b_sandbox.py index e01b9120416..e58392ca266 100644 --- a/tests/unit/sandbox/test_e2b_sandbox.py +++ b/tests/unit/sandbox/test_e2b_sandbox.py @@ -7,6 +7,7 @@ integration tests live in tests/integration/sandbox/test_e2b_sandbox.py. """ import json +from typing import Final import httpx import pytest @@ -19,19 +20,18 @@ from litellm.llms.e2b.sandbox.transformation import ( ) +def _line_stream(lines: list[str]) -> httpx.Response: + return httpx.Response(200, stream=httpx.ByteStream("".join(f"{line}\n" for line in lines).encode())) + + class FakeResponse: - def __init__(self, *, json_data=None, lines=None, status_code=200): + def __init__(self, *, json_data: object = None, status_code: int = 200) -> None: self._json = json_data - self._lines = lines or [] self.status_code = status_code def json(self): return self._json - async def aiter_lines(self): - for line in self._lines: - yield line - class FakeHTTPClient: """Records outbound requests and returns canned responses keyed by URL.""" @@ -61,7 +61,7 @@ class FakeHTTPClient: if url.endswith("/execute"): if self.execute_raises is not None: raise self.execute_raises - return FakeResponse(lines=self.execute_lines) + return _line_stream(self.execute_lines) raise AssertionError(f"unexpected POST {url}") async def delete(self, url, headers=None, **kwargs): @@ -249,6 +249,20 @@ def test_parse_lines_skips_non_json_lines(): assert result.error is None +@pytest.mark.asyncio +async def test_run_code_keeps_unicode_line_separators_inside_stdout() -> None: + separated_text: Final = "before\u2028between\u2029after\n" + client: Final = FakeHTTPClient( + execute_lines=[json.dumps({"type": "stdout", "text": separated_text, "timestamp": 1}, ensure_ascii=False)] + ) + handle: Final = ContainerHandle(id="sbx_sep", provider="e2b", domain="e2b.app") + handle._hidden_params = {"envd_access_token": "tok"} + + result: Final = await E2BSandboxConfig().arun_code(container=handle, code="print(text)", client=client) + + assert result.stdout == separated_text + + @pytest.mark.asyncio async def test_run_code_aborts_on_output_over_cap(): big_line = "x" * (MAX_OUTPUT_BYTES + 1) diff --git a/tests/unit/sandbox/test_opensandbox_sandbox.py b/tests/unit/sandbox/test_opensandbox_sandbox.py index 2928dea100e..36ae848786c 100644 --- a/tests/unit/sandbox/test_opensandbox_sandbox.py +++ b/tests/unit/sandbox/test_opensandbox_sandbox.py @@ -27,10 +27,13 @@ def sse(data): return f"data: {json.dumps(data)}" +def _line_stream(lines: list[str]) -> httpx.Response: + return httpx.Response(200, stream=httpx.ByteStream("".join(f"{line}\n" for line in lines).encode())) + + class FakeResponse: - def __init__(self, *, json_data=None, lines=None, status_code=200): + def __init__(self, *, json_data: object = None, status_code: int = 200) -> None: self._json = json_data - self._lines = lines or [] self.status_code = status_code def json(self): @@ -40,10 +43,6 @@ class FakeResponse: if self.status_code >= 400: raise http_status_error(self.status_code) - async def aiter_lines(self): - for line in self._lines: - yield line - class FakeHTTPClient: def __init__( @@ -93,7 +92,7 @@ class FakeHTTPClient: if url.endswith("/code"): if self.execute_raises is not None: raise self.execute_raises - return FakeResponse(lines=self.execute_lines) + return _line_stream(self.execute_lines) raise AssertionError(f"unexpected POST {url}") async def get(self, url, headers=None, params=None, **kwargs): diff --git a/tests/unit/test_groq_streaming_encoding.py b/tests/unit/test_groq_streaming_encoding.py index ffe87fb9890..37f05272f67 100644 --- a/tests/unit/test_groq_streaming_encoding.py +++ b/tests/unit/test_groq_streaming_encoding.py @@ -6,148 +6,58 @@ UTF-8 encoded content in streaming responses, specifically fixing the ASCII encoding error described in issue #12660. """ -import asyncio -from unittest.mock import AsyncMock, Mock +import json +from typing import Final +from unittest.mock import Mock +import httpx import pytest +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.openai_like.chat.handler import make_call, make_sync_call - -class MockResponse: - """Mock httpx response for testing UTF-8 handling.""" - - def __init__(self, test_content: str): - self.test_content = test_content - self.status_code = 200 - - def iter_text(self, encoding="utf-8"): - """Mock iter_text that yields content with the specified encoding.""" - yield self.test_content - - async def aiter_text(self, encoding="utf-8"): - """Mock aiter_text that yields content with the specified encoding.""" - yield self.test_content - - def iter_lines(self): - """Mock iter_lines method for synchronous streaming.""" - yield self.test_content - - async def aiter_lines(self): - """Mock aiter_lines method for asynchronous streaming.""" - yield self.test_content - - def json(self): - return {"choices": [{"delta": {"content": "test"}}]} +_UNICODE_TEXTS: Final = ( + "The symbol µ represents micro", + "Testing © character", + "Testing ™ character", + "Testing € character", + "Testing 北京 character", + "Testing 🚀 character", + "Testing Ñoño character", +) -class MockSyncClient: - """Mock synchronous HTTP client for testing.""" - - def __init__(self, response_content: str): - self.response_content = response_content - - def post(self, *args, **kwargs): - return MockResponse(self.response_content) +def _utf8_stream_transport(text: str) -> httpx.MockTransport: + chunk: Final = {"choices": [{"index": 0, "delta": {"content": text}}]} + body: Final = f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n".encode() + return httpx.MockTransport(lambda request: httpx.Response(200, stream=httpx.ByteStream(body))) -class MockAsyncClient: - """Mock asynchronous HTTP client for testing.""" - - def __init__(self, response_content: str): - self.response_content = response_content - - async def post(self, *args, **kwargs): - return MockResponse(self.response_content) +def _call_args() -> dict[str, object]: + return { + "api_base": "https://test.com/v1/chat/completions", + "headers": {"Authorization": "Bearer test"}, + "data": '{"model": "test", "messages": []}', + "model": "test-model", + "messages": [], + "logging_obj": Mock(), + } -def test_utf8_streaming_sync(): - """Test that synchronous streaming handles UTF-8 characters correctly.""" - # Content with the µ character that was causing issues - test_content = ( - 'data: {"choices":[{"delta":{"content":"The symbol µ represents micro"}}]}\n\n' +@pytest.mark.parametrize("text", _UNICODE_TEXTS) +def test_utf8_streaming_sync(text: str) -> None: + completion_stream: Final = make_sync_call( + client=HTTPHandler(client=httpx.Client(transport=_utf8_stream_transport(text))), **_call_args() ) - mock_client = MockSyncClient(test_content) - mock_logging = Mock() - - # This should not raise an ASCII encoding error - completion_stream = make_sync_call( - client=mock_client, - api_base="https://test.com/v1/chat/completions", - headers={"Authorization": "Bearer test"}, - data='{"model": "test", "messages": []}', - model="test-model", - messages=[], - logging_obj=mock_logging, - ) - - # Verify we can iterate through the stream without encoding errors - assert completion_stream is not None + assert "".join(chunk["text"] for chunk in completion_stream) == text @pytest.mark.asyncio -async def test_utf8_streaming_async(): - """Test that asynchronous streaming handles UTF-8 characters correctly.""" - # Content with the µ character that was causing issues - test_content = ( - 'data: {"choices":[{"delta":{"content":"The symbol µ represents micro"}}]}\n\n' +@pytest.mark.parametrize("text", _UNICODE_TEXTS) +async def test_utf8_streaming_async(text: str) -> None: + completion_stream: Final = await make_call( + client=AsyncHTTPHandler(transport=_utf8_stream_transport(text)), **_call_args() ) - mock_client = MockAsyncClient(test_content) - mock_logging = Mock() - - # This should not raise an ASCII encoding error - completion_stream = await make_call( - client=mock_client, - api_base="https://test.com/v1/chat/completions", - headers={"Authorization": "Bearer test"}, - data='{"model": "test", "messages": []}', - model="test-model", - messages=[], - logging_obj=mock_logging, - ) - - # Verify we can iterate through the stream without encoding errors - assert completion_stream is not None - - -def test_various_unicode_characters(): - """Test streaming with various Unicode characters that could cause issues.""" - unicode_test_cases = [ - "µ", # Micro symbol (the original issue) - "©", # Copyright symbol - "™", # Trademark symbol - "€", # Euro symbol - "北京", # Chinese characters - "🚀", # Emoji - "Ñoño", # Spanish characters with tildes - ] - - for unicode_char in unicode_test_cases: - test_content = f'data: {{"choices":[{{"delta":{{"content":"Testing {unicode_char} character"}}}}]}}\n\n' - - mock_client = MockSyncClient(test_content) - mock_logging = Mock() - - # This should not raise an ASCII encoding error for any Unicode character - completion_stream = make_sync_call( - client=mock_client, - api_base="https://test.com/v1/chat/completions", - headers={"Authorization": "Bearer test"}, - data='{"model": "test", "messages": []}', - model="test-model", - messages=[], - logging_obj=mock_logging, - ) - - assert ( - completion_stream is not None - ), f"Failed to handle Unicode character: {unicode_char}" - - -if __name__ == "__main__": - test_utf8_streaming_sync() - asyncio.run(test_utf8_streaming_async()) - test_various_unicode_characters() - print("All UTF-8 streaming tests passed!") + assert "".join([chunk["text"] async for chunk in completion_stream]) == text From 7650f766b006077442abe758ee4b22b5c436ad18 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Wed, 28 Oct 2026 17:01:52 -0700 Subject: [PATCH 26/27] refactor(router): fence the router surface and document the router-core boundary (#48839) Add rustdoc overviews for a new litellm-router-core crate and for litellm-model-catalog, a module docstring on litellm/router.py, and a shrink-only allowlist of the Router members and router_utils modules used outside the router, checked in CI by router_surface_check.py. Move the router-free helpers out of litellm/router_utils: batch_utils to litellm_core_utils, and the auto-router naming contract, tuning baseline and baseline request next to the auto-router plugin. No behaviour change. --- .github/workflows/test-linting.yml | 3 + litellm-rust/Cargo.lock | 4 + litellm-rust/crates/model-catalog/src/lib.rs | 33 +++ litellm-rust/crates/router-core/Cargo.toml | 9 + litellm-rust/crates/router-core/src/lib.rs | 63 +++++ .../batch_utils.py | 0 .../prompt_templates/common_utils.py | 2 +- .../llms/anthropic/prompt_cache_prediction.py | 2 +- litellm/proxy/common_request_processing.py | 2 +- .../guardrails/auto_router_compression.py | 2 +- litellm/proxy/health_check.py | 2 +- .../proxy/hooks/autorouter_baseline_cache.py | 2 +- .../proxy/hooks/model_max_budget_limiter.py | 2 +- litellm/proxy/litellm_pre_call_utils.py | 2 +- .../auto_router_endpoints.py | 12 +- .../model_management_endpoints.py | 31 +-- .../auto_router_availability.py | 4 +- .../auto_router_permissions.py | 5 +- litellm/proxy/proxy_server.py | 14 +- litellm/router.py | 61 +++-- .../auto_router}/auto_router_model_naming.py | 0 .../auto_router_tuning_baseline.py | 0 .../auto_router}/baseline_request.py | 0 litellm/router_strategy/budget_limiter.py | 2 +- litellm/router_strategy/least_busy.py | 2 +- litellm/router_strategy/lowest_cost.py | 2 +- litellm/router_strategy/lowest_latency.py | 2 +- litellm/router_strategy/lowest_tpm_rpm.py | 2 +- litellm/router_strategy/lowest_tpm_rpm_v2.py | 2 +- .../router_utils/fallback_event_handlers.py | 2 +- .../router_code_coverage.py | 4 +- .../router_surface_allowlist.json | 211 ++++++++++++++++ .../router_surface_check.py | 235 ++++++++++++++++++ .../unbounded_in_baseline.txt | 2 +- .../test_batch_utils.py | 13 +- .../test_auto_router_endpoints.py | 4 +- .../test_model_management_endpoints.py | 49 ++-- .../test_auto_router_availability.py | 7 +- .../unit/proxy/proxy_server/test_lifecycle.py | 12 +- .../test_auto_router_model_naming.py | 8 +- .../test_auto_router_tuning_baseline.py | 4 +- .../test_baseline_request.py | 2 +- .../router_strategy/test_complexity_router.py | 36 ++- tests/unit/router_strategy/test_llm_v2.py | 4 +- tests/unit/test_private_usage_aliases.py | 2 +- .../test_router/test_router_helper_utils.py | 15 +- .../test_router/test_router_surface_check.py | 58 +++++ 47 files changed, 797 insertions(+), 138 deletions(-) create mode 100644 litellm-rust/crates/router-core/Cargo.toml create mode 100644 litellm-rust/crates/router-core/src/lib.rs rename litellm/{router_utils => litellm_core_utils}/batch_utils.py (100%) rename litellm/{router_utils => router_strategy/auto_router}/auto_router_model_naming.py (100%) rename litellm/{router_utils => router_strategy/auto_router}/auto_router_tuning_baseline.py (100%) rename litellm/{router_utils => router_strategy/auto_router}/baseline_request.py (100%) create mode 100644 tests/code_coverage_tests/router_surface_allowlist.json create mode 100644 tests/code_coverage_tests/router_surface_check.py rename tests/unit/{router_utils => litellm_core_utils}/test_batch_utils.py (97%) rename tests/unit/{router_utils => router_strategy}/test_auto_router_model_naming.py (99%) rename tests/unit/{router_utils => router_strategy}/test_auto_router_tuning_baseline.py (99%) rename tests/unit/{router_utils => router_strategy}/test_baseline_request.py (95%) create mode 100644 tests/unit/test_router/test_router_surface_check.py diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index b9f8d7e7dfb..89ca60ad9d1 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -561,6 +561,9 @@ jobs: - name: router_code_coverage run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py + - name: router_surface_check + run: uv run --no-sync python ./tests/code_coverage_tests/router_surface_check.py + - name: test_chat_completion_imports run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 9c45c5c4b1c..10e590b5006 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4393,6 +4393,10 @@ dependencies = [ "rstest", ] +[[package]] +name = "litellm-router-core" +version = "0.1.0" + [[package]] name = "litellm-router-types" version = "0.1.0" diff --git a/litellm-rust/crates/model-catalog/src/lib.rs b/litellm-rust/crates/model-catalog/src/lib.rs index 066c2c83c6b..d42c05e6fe7 100644 --- a/litellm-rust/crates/model-catalog/src/lib.rs +++ b/litellm-rust/crates/model-catalog/src/lib.rs @@ -1,3 +1,36 @@ +//! The model catalog: per-model pricing, context limits and capabilities. +//! +//! This is the Rust form of `model_prices_and_context_window.json`, the map Python reads as +//! `litellm.model_cost` and through `litellm.get_model_info`. It answers "what does this model +//! cost and what can it do"; it knows nothing about configured deployments. The router's view of +//! deployments (groups, aliases, wildcard patterns, team mappings) is the router registry, defined +//! in `litellm-router-core`, which reads this catalog through its dynamic config. +//! +//! # Structure +//! - `Catalog`: the parsed map. `Catalog::parse` takes the JSON body and a `Provenance` (where +//! the body came from: the bundled copy in the repo or the remote copy, with revision and etag), +//! and keeps every entry in file order with its raw fields. +//! - `ModelEntry`: one model. `ModelEntry::fields` is the raw JSON object, so unknown fields are +//! never lost; `ModelEntry::info` is the typed mirror `ModelInfo` (pricing in `pricing`, +//! modalities and modes in `capabilities`). +//! - Aliases: an entry may list `aliases`; each alias resolves to its canonical key. Problems +//! found while indexing them (bad lists, collisions) are kept as `AliasIssue`s instead of +//! failing the parse. +//! - `FallbackGeneralizations` / `FallbackRule`: regex rules that map unknown model ids onto known +//! families. They are data in the file, not entries. +//! - Validation: `Catalog::validate` checks a catalog against `IntegrityLimits` (a remote copy that +//! lost most of its models is refused), and `validate_registry` checks every entry against the +//! checked-in contract. +//! +//! # Lookups +//! `Catalog::lookup` resolves a key exactly first (entry or alias), then case-insensitively, and +//! returns a `ModelMatch` with both the key that matched and the canonical key of the entry. Provider +//! prefixes are not stripped here; callers decide which keys to try. +//! +//! # Readers +//! Cost tracking, the router (through `RouterDynamicConfig`) and the gateway. Nothing in this crate +//! performs I/O; loading and refreshing the file is the caller's job. + mod capabilities; mod catalog; mod error; diff --git a/litellm-rust/crates/router-core/Cargo.toml b/litellm-rust/crates/router-core/Cargo.toml new file mode 100644 index 00000000000..af97bd52c57 --- /dev/null +++ b/litellm-rust/crates/router-core/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "litellm-router-core" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "The router core behind litellm.Router: registry, settings, health, selection and the call loop" + +[dependencies] diff --git a/litellm-rust/crates/router-core/src/lib.rs b/litellm-rust/crates/router-core/src/lib.rs new file mode 100644 index 00000000000..05d95d02c84 --- /dev/null +++ b/litellm-rust/crates/router-core/src/lib.rs @@ -0,0 +1,63 @@ +//! Routing for LiteLLM: decide which deployment serves a request, and drive retries, fallbacks +//! and cooldowns around the attempts. +//! +//! This crate defines the boundary `litellm.Router` is moving behind. Today the Python router in +//! `litellm/router.py` implements all of it; this crate is where the Rust core will implement the +//! same interface. Until then it holds this overview and nothing else, so the interface is written +//! down in code before either side changes. +//! +//! # The public surface +//! Code outside the router uses `litellm.Router` through two kinds of members and nothing else: +//! - The endpoint methods (`acompletion`, `aembedding`, `aresponses`, `aanthropic_messages`, the +//! files, batches, assistants and vector store wrappers, 37 in all). Each is a thin wrapper over +//! `call(op, request)`. +//! - Eight entry points: `registry()`, `settings()`, `health()`, `select()`, `apply()`, +//! `observe()`, `register_plugin()` and `retire()`. +//! +//! Every other member that outside code uses today has a named replacement among those. The old +//! names stay available for about a month through a `DeprecatedRouterMembers` mixin, which +//! implements them over the entry points and warns on use. A CI check +//! (`tests/code_coverage_tests/router_surface_check.py`) keeps an allowlist of the members outside +//! code touches; the list may only shrink. +//! +//! # Structure +//! - `Registry`: the configured deployments, indexed by id, model group, alias, wildcard pattern +//! and team. Immutable and versioned. Seven queries: `deployment(id)`, `deployments(filter)`, +//! `names(filter)`, `resolve(name, scope)`, `group(name)`, `access_groups()` and +//! `credentials(id)`. "Registry" is the router's view of what is configured; per-model pricing +//! and capabilities are the model catalog (`litellm-model-catalog`), which the router reads +//! through its dynamic config. +//! - `RouterConfig` / `RouterDynamicConfig`: everything the router needs to know, passed in. The +//! router never reads process globals. Static config (model list, strategy, retry, timeout and +//! cooldown policy, Redis settings, plugins) is fixed for the router's life. Dynamic config +//! (pricing, model aliases, fallbacks, retry counts, callbacks) is a versioned snapshot read +//! once per call. +//! - `Settings`: one typed, frozen view of the configuration a caller may read. +//! - `Health`: cooldowns, blocked deployments and usage, kept in shared cache keys so Python and +//! Rust workers see the same state. `report(id, outcome)` records outcomes the router did not +//! see itself (a health check, an interceptor that marks a deployment failed). +//! - `select(model, request)`: resolve a name, filter eligible deployments, pick one with the +//! configured strategy, without calling it. +//! - `apply(changes)`: one `ChangeSet` (upserts, deletes, settings), applied atomically, producing +//! a new registry version. +//! - `call(op, request)`: select, run the attempt, and retry or fall back until it succeeds or the +//! policy is exhausted. +//! +//! # Execution +//! Each attempt resolves an `Executor` after a deployment is picked. A Rust-native operation runs +//! its `inference-*` route machine in-process; anything else goes through `Host::invoke`. A +//! started attempt is never replayed on a different executor. +//! +//! # Hosts +//! `Host` is what the embedding process provides: `invoke` for non-native operations, `select` +//! for strategies and sub-routers not yet ported, `allow_fallback` for access and budget checks, +//! `sleep`, `observe` for routing events, and `cache`, a DualCache-style store that holds all +//! routing state under the same keys, encodings and TTLs the Python router uses. `PythonHost` +//! serves them from the existing Python code when the router runs inside Python; `NativeHost` +//! serves them in the Rust server, where an operation without a Rust implementation makes the +//! config fail at build time, never partway through a request. +//! +//! # Plugins +//! Routing strategies, pre-call checks and sub-routers (auto, complexity, quality, adaptive) +//! register through `register_plugin` and see a `RoutingContext` (registry, settings, health and +//! the current request), never the router itself. diff --git a/litellm/router_utils/batch_utils.py b/litellm/litellm_core_utils/batch_utils.py similarity index 100% rename from litellm/router_utils/batch_utils.py rename to litellm/litellm_core_utils/batch_utils.py diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index cf6bdc69556..101f069c0f8 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -21,8 +21,8 @@ from pydantic import TypeAdapter import litellm from litellm import verbose_logger +from litellm.litellm_core_utils.batch_utils import InMemoryFile from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.router_utils.batch_utils import InMemoryFile from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index 7a4c707860d..17c178d169e 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -29,7 +29,7 @@ from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils, prepare_native_messages -from litellm.router_utils.baseline_request import ( +from litellm.router_strategy.auto_router.baseline_request import ( BASELINE_PARAMETERS, capture_baseline_parameters, ) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 70e305f46fb..9bb451495c8 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -139,8 +139,8 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports ) from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task from litellm.router import Router +from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict -from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model from litellm.types.extraction import ExtractionResponse from litellm.types.guardrails import GuardrailEventHooks diff --git a/litellm/proxy/guardrails/auto_router_compression.py b/litellm/proxy/guardrails/auto_router_compression.py index 1267fb15db8..3071bcbb1dc 100644 --- a/litellm/proxy/guardrails/auto_router_compression.py +++ b/litellm/proxy/guardrails/auto_router_compression.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Final from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket -from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX +from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 0d2f8dc9a0c..be32405154c 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -27,7 +27,7 @@ from litellm.constants import ( HEALTH_CHECK_TIMEOUT_SECONDS, ) from litellm.litellm_core_utils.health_check_helpers import native_health_check_mode -from litellm.router_utils.auto_router_model_naming import ( +from litellm.router_strategy.auto_router.auto_router_model_naming import ( StrategyRouterDependency, classify_strategy_router_model, strategy_router_dependencies, diff --git a/litellm/proxy/hooks/autorouter_baseline_cache.py b/litellm/proxy/hooks/autorouter_baseline_cache.py index c4232609124..1147ce7ec12 100644 --- a/litellm/proxy/hooks/autorouter_baseline_cache.py +++ b/litellm/proxy/hooks/autorouter_baseline_cache.py @@ -37,8 +37,8 @@ from litellm.proxy.spend_tracking.savings import ( _effective_model_info, # pyright: ignore[reportPrivateUsage] # existing deployment-price owner _proxy_llm_router, # pyright: ignore[reportPrivateUsage] # existing optional proxy-router owner ) +from litellm.router_strategy.auto_router.baseline_request import baseline_request from litellm.router_strategy.complexity_router.context_compaction import compaction_applied -from litellm.router_utils.baseline_request import baseline_request from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.router import BaselineRouteStamp from litellm.types.utils import CallTypes, ModelInfo, Usage diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index 11a47b2d424..1c4a61cfad8 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -19,6 +19,7 @@ from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import Span +from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.llms.bedrock.common_utils import get_bedrock_base_model, get_bedrock_cross_region_inference_regions @@ -30,7 +31,6 @@ from litellm.proxy.hooks.model_budget_shared_windows import ( SharedWindows, ) from litellm.router_strategy.budget_limiter import RouterBudgetLimiting, budget_window_seconds -from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import BudgetConfig, LlmProviders, StandardLoggingPayload diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index d4528bc85e8..2c05f724374 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -3290,7 +3290,7 @@ def _per_user_credential_names_for_groups( GITHUB_COPILOT_AUTH_TYPE_KEY, GITHUB_COPILOT_PER_USER_AUTH_TYPE, ) - from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model + from litellm.router_strategy.auto_router.auto_router_model_naming import classify_strategy_router_model names: Final[list[str]] = [] # mutable-ok: accumulates one name per per-user deployment for group in model_groups: diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 4f48f3c57b9..3dceab6b7f7 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -60,16 +60,16 @@ from litellm.repositories.table_repositories import PrismaTableRepository from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository from litellm.repositories.verification_token_repository import VerificationTokenRepository +from litellm.router_strategy.auto_router.auto_router_model_naming import ( + StrategyRouterDependencyRole, + classify_strategy_router_model, + strategy_router_dependencies, +) from litellm.router_strategy.complexity_router import ComplexityRouter from litellm.router_strategy.complexity_router.request_models import ( AutoRouterRoutingTestRequest, RequestComplexityRouterConfig, ) -from litellm.router_utils.auto_router_model_naming import ( - StrategyRouterDependencyRole, - classify_strategy_router_model, - strategy_router_dependencies, -) from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.management_endpoints.auto_router_endpoints import ( SHADOW_EVAL_TURN_VALVE, @@ -411,7 +411,7 @@ async def validate_complexity_router_config( auto_router_environment_reference_error(data.complexity_router_config, user_api_key_dict) ) - from litellm.router_utils.auto_router_model_naming import ( + from litellm.router_strategy.auto_router.auto_router_model_naming import ( validate_complexity_router_config_write, ) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 45006b14df6..8cd3559ba86 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -131,6 +131,23 @@ from litellm.repositories.project_repository import ProjectRepository from litellm.repositories.table_repositories import ModelTableRepository from litellm.repositories.team_repository import TeamRepository from litellm.router import Router +from litellm.router_strategy.auto_router.auto_router_model_naming import ( + GATED_AUTO_ROUTER_CAPABILITIES, + STRATEGY_ROUTER_PARAM_FIELDS, + GatedAutoRouterCapability, + capability_limit_violation, + carries_complexity_router_settings, + count_capability_routers, + gated_capabilities_of, + is_complexity_router_model, + validate_complexity_router_config_placement, + validate_complexity_router_config_write, + validate_strategy_router_model_write, +) +from litellm.router_strategy.auto_router.auto_router_tuning_baseline import ( + is_mutable_tuned_candidate, + tuning_quota_violation, +) from litellm.router_strategy.complexity_router import ( DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE, ClassificationRubric, @@ -144,20 +161,6 @@ from litellm.router_strategy.complexity_router import ( normalize_classification_prompt, ) from litellm.router_strategy.complexity_router.config import resolve_complexity_router_config_write -from litellm.router_utils.auto_router_model_naming import ( - GATED_AUTO_ROUTER_CAPABILITIES, - STRATEGY_ROUTER_PARAM_FIELDS, - GatedAutoRouterCapability, - capability_limit_violation, - carries_complexity_router_settings, - count_capability_routers, - gated_capabilities_of, - is_complexity_router_model, - validate_complexity_router_config_placement, - validate_complexity_router_config_write, - validate_strategy_router_model_write, -) -from litellm.router_utils.auto_router_tuning_baseline import is_mutable_tuned_candidate, tuning_quota_violation from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.bedrock import AwsSessionTag from litellm.types.proxy.management_endpoints.model_management_endpoints import ( diff --git a/litellm/proxy/management_helpers/auto_router_availability.py b/litellm/proxy/management_helpers/auto_router_availability.py index 9b511ce0817..ce7859aff2d 100644 --- a/litellm/proxy/management_helpers/auto_router_availability.py +++ b/litellm/proxy/management_helpers/auto_router_availability.py @@ -7,14 +7,14 @@ from typing import Final from pydantic import Json, TypeAdapter, ValidationError from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper -from litellm.router_utils.auto_router_model_naming import ( +from litellm.router_strategy.auto_router.auto_router_model_naming import ( GATED_AUTO_ROUTER_CAPABILITIES, capability_limit_violation, classify_strategy_router_model, count_capability_routers, gated_capabilities_of, ) -from litellm.router_utils.auto_router_tuning_baseline import ( +from litellm.router_strategy.auto_router.auto_router_tuning_baseline import ( is_mutable_tuned_candidate, mutable_tuned_identities, tuning_quota_violation, diff --git a/litellm/proxy/management_helpers/auto_router_permissions.py b/litellm/proxy/management_helpers/auto_router_permissions.py index 995446db7a6..8597f0d3667 100644 --- a/litellm/proxy/management_helpers/auto_router_permissions.py +++ b/litellm/proxy/management_helpers/auto_router_permissions.py @@ -34,13 +34,16 @@ from litellm.repositories.prisma_protocols import DatabaseClient from litellm.repositories.project_repository import ProjectRepository from litellm.repositories.table_repositories import TeamMembershipRepository from litellm.router import Router +from litellm.router_strategy.auto_router.auto_router_model_naming import ( + classify_strategy_router_model, + strategy_router_dependencies, +) from litellm.router_strategy.complexity_router.config import ( ComplexityRouterConfigWrite, normalize_classifier_config_aliases, resolve_complexity_router_config_write, ) from litellm.router_strategy.complexity_router.request_models import RequestComplexityRouterConfig -from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model, strategy_router_dependencies from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.router import Deployment, updateDeployment diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 70e1c25a70a..b263412ce0f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -168,12 +168,7 @@ from litellm.proxy.common_utils.token_count_settings import ( ) from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields -from litellm.router_utils.access_windows import access_windows_config_error -from litellm.router_utils.add_retry_fallback_headers import ( - get_fallback_errors_from_headers, - get_hidden_params_dict, -) -from litellm.router_utils.auto_router_model_naming import ( +from litellm.router_strategy.auto_router.auto_router_model_naming import ( GATED_AUTO_ROUTER_CAPABILITIES, STRATEGY_ROUTER_PARAM_FIELDS, capability_limit_violation, @@ -181,12 +176,17 @@ from litellm.router_utils.auto_router_model_naming import ( count_capability_routers, validate_complexity_router_config_placement, ) -from litellm.router_utils.auto_router_tuning_baseline import ( +from litellm.router_strategy.auto_router.auto_router_tuning_baseline import ( TUNING_BASELINE_PARAM_NAME, mutable_tuned_identities, snapshot_tuning_baselines, tuning_limit_violation, ) +from litellm.router_utils.access_windows import access_windows_config_error +from litellm.router_utils.add_retry_fallback_headers import ( + get_fallback_errors_from_headers, + get_hidden_params_dict, +) from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.router_utils.routing_groups import parse_routing_groups from litellm.types.caching import RedisPipelineIncrementOperation diff --git a/litellm/router.py b/litellm/router.py index 408cb20945f..fc888495e99 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7,6 +7,29 @@ # # Thank you ! We ❤️ you! - Krrish & Ishaan +""" +litellm.Router: load balancing, retries, fallbacks and cooldowns across deployments. + +Public surface. The router is moving behind a narrow interface; code outside +the router should end up using a Router through two kinds of members only: + +- endpoint methods (acompletion, aembedding, aresponses, aanthropic_messages, + the files, batches, assistants and vector store wrappers, ...), each a thin + wrapper around one call; +- entry points (being added) that replace direct access to internals: registry() for + deployment lookups, settings() for configuration reads, health() for + cooldowns, blocks and usage, select() to pick a deployment without calling + it, apply() for runtime changes, observe() for routing events, + register_plugin() for strategies and sub-routers, and retire() to release + the router. + +Everything else on the class is internal. The members outside code uses today +are listed in tests/code_coverage_tests/router_surface_allowlist.json, and a CI +check (router_surface_check.py next to it) fails on any new outside use, so the +list only shrinks while callers move to the entry points. The same interface +is documented for the Rust core in litellm-rust/crates/router-core. +""" + import asyncio import contextlib import copy @@ -85,6 +108,15 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.routing import routing_decision_attributes from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span from litellm.litellm_core_utils.asyncify import run_async_function +from litellm.litellm_core_utils.batch_utils import ( + batch_create_routing, + deployment_supported_endpoints, + get_router_metadata_variable_name, + is_batch_retrieve_call_type, + replace_model_in_jsonl, + should_replace_model_in_jsonl, + without_deployments_that_cannot_batch, +) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import ( coerce_token_limit, @@ -148,6 +180,15 @@ from litellm.llms.openai_like.model_info import ( get_openai_compatible_model_info, ) from litellm.responses.bridge_check import keep_registered_mode +from litellm.router_strategy.auto_router.auto_router_model_naming import ( + AUTO_ROUTER_MODEL_PREFIX, + STRATEGY_ROUTER_PARAM_FIELDS, + GatedAutoRouterCapability, + capability_limit_violation, + claimed_capabilities, + classify_strategy_router_model, + count_capability_routers, +) from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.router_strategy.complexity_router.context_compaction import ( @@ -187,24 +228,6 @@ from litellm.router_utils.add_retry_fallback_headers import ( replace_complexity_router_headers, response_total_token_count, ) -from litellm.router_utils.auto_router_model_naming import ( - AUTO_ROUTER_MODEL_PREFIX, - STRATEGY_ROUTER_PARAM_FIELDS, - GatedAutoRouterCapability, - capability_limit_violation, - claimed_capabilities, - classify_strategy_router_model, - count_capability_routers, -) -from litellm.router_utils.batch_utils import ( - batch_create_routing, - deployment_supported_endpoints, - get_router_metadata_variable_name, - is_batch_retrieve_call_type, - replace_model_in_jsonl, - should_replace_model_in_jsonl, - without_deployments_that_cannot_batch, -) from litellm.router_utils.client_initalization_utils import InitalizeCachedClient, MaxParallelRequestsLimit from litellm.router_utils.clientside_credential_handler import ( get_dynamic_litellm_params, @@ -16271,7 +16294,7 @@ class Router: to the deployment that actually served the request. Every attempt therefore writes or clears, never just writes. """ - from litellm.router_utils.baseline_request import capture_baseline_parameters + from litellm.router_strategy.auto_router.baseline_request import capture_baseline_parameters from litellm.types.router import BaselineRouteStamp phase_attributes(routing_decision_attributes(routing_decision)) diff --git a/litellm/router_utils/auto_router_model_naming.py b/litellm/router_strategy/auto_router/auto_router_model_naming.py similarity index 100% rename from litellm/router_utils/auto_router_model_naming.py rename to litellm/router_strategy/auto_router/auto_router_model_naming.py diff --git a/litellm/router_utils/auto_router_tuning_baseline.py b/litellm/router_strategy/auto_router/auto_router_tuning_baseline.py similarity index 100% rename from litellm/router_utils/auto_router_tuning_baseline.py rename to litellm/router_strategy/auto_router/auto_router_tuning_baseline.py diff --git a/litellm/router_utils/baseline_request.py b/litellm/router_strategy/auto_router/baseline_request.py similarity index 100% rename from litellm/router_utils/baseline_request.py rename to litellm/router_strategy/auto_router/baseline_request.py diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index eab232eae18..1d502dd8d95 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -36,12 +36,12 @@ from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache from litellm.caching.redis_cache import RedisCache, RedisPipelineIncrementOperation, log_redis_failure from litellm.integrations.custom_logger import CustomLogger, Span +from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, ) from litellm.litellm_core_utils.duration_parser import duration_in_seconds_from from litellm.router_strategy.tag_based_routing import get_tags_from_request_kwargs -from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once from litellm.router_utils.cooldown_callbacks import ( get_prometheus_logger_from_callbacks, ) diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py index 6f3e0936641..8e0c3d6a7e7 100644 --- a/litellm/router_strategy/least_busy.py +++ b/litellm/router_strategy/least_busy.py @@ -10,7 +10,7 @@ from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache from litellm.caching.redis_cache import log_redis_failure from litellm.integrations.custom_logger import CustomLogger -from litellm.router_utils.batch_utils import is_batch_retrieve_call_type +from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type IN_FLIGHT_COUNT_TTL_SECONDS: Final = 60 * 60 diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index 5894da9cb4f..17d03fd4ff8 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -10,7 +10,7 @@ from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger -from litellm.router_utils.batch_utils import is_batch_retrieve_call_type +from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 4e994d3fdc6..c352934db6c 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -13,9 +13,9 @@ from litellm import ModelResponse, token_counter, verbose_logger from litellm._internal_context import with_service_target from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs, safe_divide_seconds from litellm.responses.utils import ResponseAPILoggingUtils -from litellm.router_utils.batch_utils import is_batch_retrieve_call_type from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import LiteLLMPydanticObjectBase diff --git a/litellm/router_strategy/lowest_tpm_rpm.py b/litellm/router_strategy/lowest_tpm_rpm.py index 2d373e0c266..9636d7f009d 100644 --- a/litellm/router_strategy/lowest_tpm_rpm.py +++ b/litellm/router_strategy/lowest_tpm_rpm.py @@ -9,7 +9,7 @@ from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger -from litellm.router_utils.batch_utils import is_batch_retrieve_call_type +from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type from litellm.types.utils import LiteLLMPydanticObjectBase from litellm.utils import print_verbose diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index 527ea830d55..0719b30d36d 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -15,8 +15,8 @@ from litellm._internal_context import with_service_target from litellm._logging import verbose_logger, verbose_router_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs -from litellm.router_utils.batch_utils import is_batch_retrieve_call_type from litellm.types.router import RouterErrors from litellm.types.utils import LiteLLMPydanticObjectBase, StandardLoggingPayload from litellm.utils import get_utc_datetime, print_verbose diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 0a77ad5b9ea..6d1de1f9f6c 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -13,6 +13,7 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_router_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs, safe_deep_copy from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider @@ -22,7 +23,6 @@ from litellm.router_utils.add_retry_fallback_headers import ( add_fallback_headers_to_response, get_fallback_error_info, ) -from litellm.router_utils.batch_utils import get_router_metadata_variable_name from litellm.router_utils.cooldown_handlers import ( _first_present, # pyright: ignore[reportPrivateUsage] - shared internal helper, used across router_utils cast_exception_status_to_int, diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 4c9afd26dad..9fb3633d53c 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -187,13 +187,13 @@ ignored_function_names = [ def main(): router_file = [ "./litellm/router.py", - "./litellm/router_utils/batch_utils.py", + "./litellm/litellm_core_utils/batch_utils.py", "./litellm/router_utils/pattern_match_deployments.py", ] # router_file = [ # "../../litellm/router.py", # "../../litellm/router_utils/pattern_match_deployments.py", - # "../../litellm/router_utils/batch_utils.py", + # "../../litellm/litellm_core_utils/batch_utils.py", # ] ## LOCAL TESTING tests_dir = "./tests/" # Update this path if your tests directory is located elsewhere # tests_dir = "../../tests/" # LOCAL TESTING diff --git a/tests/code_coverage_tests/router_surface_allowlist.json b/tests/code_coverage_tests/router_surface_allowlist.json new file mode 100644 index 00000000000..428fd9b4011 --- /dev/null +++ b/tests/code_coverage_tests/router_surface_allowlist.json @@ -0,0 +1,211 @@ +{ + "_doc": "Router members and litellm.router_utils modules used outside the router. Shrink-only: see tests/code_coverage_tests/router_surface_check.py.", + "members_outside": [ + "_get_client", + "_zero_cost_cache", + "a_add_message", + "aadapter_completion", + "aanthropic_messages", + "abatch_completion", + "abatch_completion_fastest_response", + "acancel_batch", + "acancel_fine_tuning_job", + "acompletion", + "acreate_assistants", + "acreate_batch", + "acreate_file", + "acreate_fine_tuning_job", + "acreate_thread", + "adaptive_routers", + "adecisions", + "adelete_assistant", + "aembedding", + "afile_content", + "afile_delete", + "aget_assistants", + "aget_messages", + "aget_responses", + "aget_thread", + "alist_batches", + "alist_fine_tuning_jobs", + "allm_passthrough_route", + "amoderation", + "aocr", + "are_all_deployments_blocked", + "arefresh_model_info", + "aretrieve_batch", + "aretrieve_fine_tuning_job", + "arm_routing_read_prefetch", + "arun_thread", + "async_get_available_deployment", + "async_get_fully_unhealthy_model_names", + "async_get_healthy_deployments", + "async_pre_routing_hook", + "auto_routers", + "avector_store_create", + "avector_store_search", + "background_health_check_model_groups", + "bind_to_request_logging", + "cache", + "cache_responses", + "cached_deployment_model_info", + "cached_model_group_info", + "completion", + "complexity_routers", + "config_deployments", + "content_policy_fallbacks", + "context_window_fallbacks", + "cooldown_time", + "default_deployment", + "default_litellm_params", + "delete_deployment", + "deployment_latency_map", + "deployment_names", + "deployment_usable_by_team", + "deployments_for_request", + "deployments_named", + "discard", + "embedding", + "enable_health_check_routing", + "fallbacks", + "generate_model_id", + "get_all_deployments", + "get_available_deployment_for_pass_through", + "get_available_guardrail", + "get_candidate_model_ids_for_route", + "get_configured_display_name", + "get_configured_mode", + "get_configured_service_tiers", + "get_configured_token_limits", + "get_credential_deployment", + "get_deployment", + "get_deployment_by_model_group_name", + "get_deployment_credentials", + "get_deployment_credentials_with_provider", + "get_deployment_model_info", + "get_discovered_model_info", + "get_fully_blocked_model_names", + "get_model_access_groups", + "get_model_from_alias", + "get_model_group_info", + "get_model_group_usage", + "get_model_ids", + "get_model_info", + "get_model_list", + "get_model_list_from_model_alias", + "get_model_list_from_routing_groups", + "get_model_list_of_routed_group", + "get_model_listing_info", + "get_model_names", + "get_remaining_model_group_usage", + "get_routable_deployments", + "get_routable_upstream_model", + "get_router_deployment_budget_limiter", + "get_routing_group", + "get_settings", + "get_valid_args", + "guardrail_list", + "has_model_id", + "health_check_ignore_transient_errors", + "health_state_cache", + "is_deployment_blocked", + "is_model_access_group_for_wildcard_route", + "is_recognized_model", + "kubernetes_pod_discovery", + "map_team_model", + "model_group_alias", + "model_list", + "model_name_to_deployment_indices", + "model_names", + "pattern_router", + "priority_header_update", + "prompt_caching_pin_cache", + "provider_budget_config", + "quality_routers", + "resolve_model_name_from_model_id", + "resolve_served_model_name", + "resolved_litellm_models", + "routable_model_group", + "router_general_settings", + "routing_groups", + "routing_plugins", + "schedule_acompletion", + "search_tools", + "serve_through_request_logging", + "should_include_deployment", + "split_request_priority", + "team_model_to_deployment_indices", + "team_pattern_routers", + "team_ptu_ceiling", + "team_public_model_names", + "team_public_model_names_of", + "update_redis_cache", + "update_settings", + "upsert_deployment", + "wildcard_route_deployments", + "wildcard_routed_models" + ], + "members_router_family": [ + "_count_pre_call_check_tokens", + "_pre_call_checks", + "_update_kwargs_before_fallbacks", + "acompletion", + "adecisions", + "aembedding", + "allowed_fails", + "allowed_fails_policy", + "aresponses", + "async_function_with_fallbacks", + "async_get_healthy_deployments", + "cache", + "cached_deployment_model_info", + "cooldown_cache", + "cooldown_time", + "default_max_parallel_requests", + "deployments_named", + "disable_cooldowns", + "embedding", + "enable_pre_call_checks", + "enable_tag_filtering", + "explicit_allowed_fails", + "fallback_access_check", + "fallback_budget_check", + "fallbacks", + "get_all_deployments", + "get_allowed_fails_from_policy", + "get_deployment", + "get_model_group", + "get_model_ids", + "get_model_info", + "get_model_list", + "get_router_model_info", + "log_retry", + "model_list", + "provider_default_deployment_ids", + "routing_group_has_alternatives", + "routing_strategy", + "search_tools", + "set_custom_routing_strategy", + "slack_alerting_logger", + "tag_filtering_match_any", + "tag_routing_prefix", + "team_model_has_alternatives" + ], + "router_utils_imports_outside": [ + "litellm.router_utils.access_windows", + "litellm.router_utils.add_retry_fallback_headers", + "litellm.router_utils.clientside_credential_handler", + "litellm.router_utils.common_utils", + "litellm.router_utils.cooldown_handlers", + "litellm.router_utils.deployment_pricing", + "litellm.router_utils.fallback_event_handlers", + "litellm.router_utils.get_retry_from_policy", + "litellm.router_utils.health_state_cache", + "litellm.router_utils.pattern_match_deployments", + "litellm.router_utils.ptu_shares", + "litellm.router_utils.reasoning_effort_capability", + "litellm.router_utils.router_callbacks.track_deployment_metrics", + "litellm.router_utils.routing_groups", + "litellm.router_utils.search_api_router" + ] +} diff --git a/tests/code_coverage_tests/router_surface_check.py b/tests/code_coverage_tests/router_surface_check.py new file mode 100644 index 00000000000..0b35bac5240 --- /dev/null +++ b/tests/code_coverage_tests/router_surface_check.py @@ -0,0 +1,235 @@ +""" +Fence for the Router's public surface. + +Code outside the router may use only the Router members listed in +router_surface_allowlist.json, and may import only the litellm.router_utils +modules listed there. The lists hold what is used today and may only shrink: + +- a member or module used outside the router that is not listed fails the + check (use an existing entry point instead, or discuss widening the surface); +- a listed member or module that nothing outside the router uses any more + also fails, so the entry is removed in the same change. + +"Outside" means every non-test file under litellm/ and enterprise/ except +litellm/router.py and the router's own modules. Uses from the router family +(litellm/router_utils, litellm/router_strategy) are listed separately, because +those modules become private parts of the router core in a later step. + +Member uses are found the same way the boundary survey found them: attribute +access on a receiver whose name looks like a router (llm_router, +litellm_router_instance, router, ...) or on the Router class, plus +getattr/setattr/hasattr with a constant name on such a receiver. Members +include the container file endpoints Router.__init__ registers by name. + +Run: python tests/code_coverage_tests/router_surface_check.py [--write] +--write regenerates the allowlist from the current tree (for shrinking it). +""" + +import ast +import json +import os +import re +import sys +from typing import Dict, List, Set + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +ALLOWLIST = os.path.join(os.path.dirname(__file__), "router_surface_allowlist.json") + +ROUTER_FILES = { + "litellm/router.py", +} +ROUTER_CLASS_FILES = ["litellm/router.py"] +ROUTER_CLASS_NAMES = {"Router"} + +SCAN_ROOTS = ["litellm", "enterprise"] +SKIP_DIRS = { + ".git", + "node_modules", + ".venv", + "__pycache__", + "litellm-rust", + "ui", + "docs", + "cookbook", + "tests", + "test", +} + +RECEIVER = re.compile( + r"(^|[._])(llm_router|router|litellm_router|litellm_router_instance|router_instance" + r"|_router|proxy_router|llm_router_instance|router_obj|mock_router|Router)$" +) +# Router.__init__ registers these with setattr(self, name, ...), so they never appear as self.. +CONTAINER_FILE_ENDPOINTS = "litellm/containers/endpoints.json" +DYNAMIC_FUNCS = {"getattr", "setattr", "hasattr"} + + +def _area(rel: str) -> str: + if rel.startswith(("litellm/router_utils/", "litellm/router_strategy/")): + return "router_family" + return "outside" + + +def _is_test(rel: str) -> bool: + name = os.path.basename(rel) + return "/tests/" in rel or name.startswith("test_") or name.endswith("_test.py") + + +def router_members() -> Set[str]: + members: Set[str] = set() + for rel in ROUTER_CLASS_FILES: + with open(os.path.join(ROOT, rel), encoding="utf-8") as f: + tree = ast.parse(f.read()) + for cls in tree.body: + if not isinstance(cls, ast.ClassDef) or cls.name not in ROUTER_CLASS_NAMES: + continue + for node in cls.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + members.add(node.name) + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + members.add(node.target.id) + elif isinstance(node, ast.Assign): + members.update(t.id for t in node.targets if isinstance(t, ast.Name)) + for node in ast.walk(cls): + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "self" + and isinstance(node.ctx, ast.Store) + ): + members.add(node.attr) + with open(os.path.join(ROOT, CONTAINER_FILE_ENDPOINTS), encoding="utf-8") as f: + for endpoint in json.load(f)["endpoints"]: + members.update((endpoint["name"], endpoint["async_name"])) + return {m for m in members if not (m.startswith("__") and m.endswith("__"))} + + +def _python_files(): + for top in SCAN_ROOTS: + base = os.path.join(ROOT, top) + for dirpath, dirnames, filenames in os.walk(base): + dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] + for fn in filenames: + if fn.endswith(".py"): + path = os.path.join(dirpath, fn) + yield os.path.relpath(path, ROOT).replace(os.sep, "/"), path + + +def scan() -> Dict[str, Dict[str, Set[str]]]: + """Return {"members": {area: {member: files}}, "router_utils_imports": {module: files}}.""" + members = router_members() + uses: Dict[str, Dict[str, Set[str]]] = {"outside": {}, "router_family": {}} + imports: Dict[str, Set[str]] = {} + for rel, path in _python_files(): + if rel in ROUTER_FILES or _is_test(rel): + continue + with open(path, encoding="utf-8") as f: + src = f.read() + has_router = "router" in src.lower() + if not has_router: + continue + try: + tree = ast.parse(src) + except SyntaxError: + continue + area = _area(rel) + for name in member_uses(tree, members): + uses[area].setdefault(name, set()).add(rel) + if area == "outside": + for node in ast.walk(tree): + for mod in _router_utils_modules(node): + imports.setdefault(mod, set()).add(rel) + return {"members": uses, "router_utils_imports": imports} + + +def member_uses(tree: ast.AST, members: Set[str]) -> Set[str]: + """The Router members a module uses, on an instance or on the Router class itself.""" + used: Set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Attribute) and node.attr in members: + if RECEIVER.search(ast.unparse(node.value)): + used.add(node.attr) + elif ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id in DYNAMIC_FUNCS + and len(node.args) >= 2 + and isinstance(node.args[1], ast.Constant) + and node.args[1].value in members + and RECEIVER.search(ast.unparse(node.args[0])) + ): + used.add(node.args[1].value) + return used + + +def _router_utils_modules(node: ast.AST) -> List[str]: + if isinstance(node, ast.ImportFrom) and node.module: + if node.module == "litellm.router_utils": + return ["litellm.router_utils." + a.name for a in node.names] + if node.module.startswith("litellm.router_utils."): + return [node.module] + if isinstance(node, ast.Import): + return [a.name for a in node.names if a.name.startswith("litellm.router_utils.")] + return [] + + +def build_allowlist(found) -> dict: + return { + "_doc": ( + "Router members and litellm.router_utils modules used outside the router. " + "Shrink-only: see tests/code_coverage_tests/router_surface_check.py." + ), + "members_outside": sorted(found["members"]["outside"]), + "members_router_family": sorted(found["members"]["router_family"]), + "router_utils_imports_outside": sorted(found["router_utils_imports"]), + } + + +def compare(allow: dict, found) -> List[str]: + errors: List[str] = [] + sections = [ + ("members_outside", found["members"]["outside"], "Router member"), + ("members_router_family", found["members"]["router_family"], "Router member (router family)"), + ("router_utils_imports_outside", found["router_utils_imports"], "litellm.router_utils import"), + ] + for key, used, what in sections: + listed = set(allow.get(key, [])) + for name in sorted(set(used) - listed): + files = ", ".join(sorted(used[name])) + errors.append( + f"new {what} outside the router: {name!r} in {files}. The router surface may only " + f"shrink: use an entry point instead." + ) + for name in sorted(listed - set(used)): + errors.append( + f"{what} {name!r} is listed in {os.path.basename(ALLOWLIST)} [{key}] but nothing uses it " + f"any more: remove it from the list." + ) + return errors + + +def main(argv: List[str]) -> int: + found = scan() + if "--write" in argv: + with open(ALLOWLIST, "w", encoding="utf-8") as f: + json.dump(build_allowlist(found), f, indent=2) + f.write("\n") + print(f"wrote {ALLOWLIST}") + return 0 + with open(ALLOWLIST, encoding="utf-8") as f: + allow = json.load(f) + errors = compare(allow, found) + for e in errors: + print(e) + if errors: + return 1 + print( + f"router surface ok: {len(allow['members_outside'])} members outside, " + f"{len(allow['members_router_family'])} from the router family, " + f"{len(allow['router_utils_imports_outside'])} router_utils modules imported outside" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/code_coverage_tests/unbounded_in_baseline.txt b/tests/code_coverage_tests/unbounded_in_baseline.txt index 1da964681a9..d8c6e2241f4 100644 --- a/tests/code_coverage_tests/unbounded_in_baseline.txt +++ b/tests/code_coverage_tests/unbounded_in_baseline.txt @@ -129,4 +129,4 @@ litellm/proxy/utils.py prefetch_config_params prisma param_name.in `param_names` litellm/repositories/daily_activity_repository.py DailyActivityRepository.daily_rows prisma [scope.entity_id_field].in `list(scope.entity_ids)` 0 litellm/repositories/daily_activity_repository.py DailyActivityRepository.daily_rows prisma api_key.in `list(scope.api_keys)` 0 litellm/repositories/daily_activity_repository.py DailyActivityRepository.daily_rows prisma not.in `list(scope.exclude_entity_ids)` 0 -litellm/router_utils/auto_router_model_naming.py raw-sql classifier_type.IN `IN ({_LLM_CLASSIFIER_TYPES_SQL})` 0 +litellm/router_strategy/auto_router/auto_router_model_naming.py raw-sql classifier_type.IN `IN ({_LLM_CLASSIFIER_TYPES_SQL})` 0 diff --git a/tests/unit/router_utils/test_batch_utils.py b/tests/unit/litellm_core_utils/test_batch_utils.py similarity index 97% rename from tests/unit/router_utils/test_batch_utils.py rename to tests/unit/litellm_core_utils/test_batch_utils.py index 79fe86a37c0..cd37d635c43 100644 --- a/tests/unit/router_utils/test_batch_utils.py +++ b/tests/unit/litellm_core_utils/test_batch_utils.py @@ -7,7 +7,7 @@ from typing import Dict, Final, List import pytest import litellm -from litellm.router_utils.batch_utils import ( +from litellm.litellm_core_utils.batch_utils import ( InMemoryFile, get_router_metadata_variable_name, parse_jsonl_with_embedded_newlines, @@ -116,7 +116,7 @@ def test_router_metadata_variable_name(): def test_non_json_input(): """Test that replace_model_in_jsonl returns original content for non-JSON input""" - from litellm.router_utils.batch_utils import replace_model_in_jsonl + from litellm.litellm_core_utils.batch_utils import replace_model_in_jsonl # Test with non-JSON string non_json_str = "This is not a JSON string" @@ -138,7 +138,7 @@ def test_non_json_input(): def test_should_replace_model_in_jsonl(): """Test that should_replace_model_in_jsonl returns the correct value""" - from litellm.router_utils.batch_utils import should_replace_model_in_jsonl + from litellm.litellm_core_utils.batch_utils import should_replace_model_in_jsonl assert should_replace_model_in_jsonl(purpose="batch") is True assert should_replace_model_in_jsonl(purpose="batch", passthrough=True) is False @@ -340,7 +340,7 @@ def test_replace_model_in_jsonl_with_embedded_newlines(): def test_is_batch_retrieve_call_type_matches_only_batch_retrieves(): - from litellm.router_utils.batch_utils import is_batch_retrieve_call_type + from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type from litellm.types.utils import CallTypes assert is_batch_retrieve_call_type(CallTypes.aretrieve_batch.value) is True @@ -389,7 +389,8 @@ def setup_and_teardown(): ) def test_batch_charge_identity_requires_a_cost_bearing_retrieval(call_type: str, cost: float, eligible: bool) -> None: from openai.types import Batch - from litellm.router_utils.batch_utils import batch_id_to_charge_once + + from litellm.litellm_core_utils.batch_utils import batch_id_to_charge_once batch: Final = Batch( id="batch-budget", completion_window="24h", created_at=1, endpoint="/v1/chat/completions", @@ -401,7 +402,7 @@ def test_batch_charge_identity_requires_a_cost_bearing_retrieval(call_type: str, def test_batch_charge_markers_keep_counters_and_batches_independent() -> None: from litellm.caching.in_memory_cache import InMemoryCache - from litellm.router_utils.batch_utils import batch_charged_once_marker_key + from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key cache: Final = InMemoryCache() cache.set_cache(batch_charged_once_marker_key("provider-spend", "first-batch"), 1) diff --git a/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py index 748b291d5ff..c13e3d5812b 100644 --- a/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py @@ -170,7 +170,7 @@ async def test_preview_and_serving_share_default_model_resolution( explicit_default: str | None, expected: str, ): - from litellm.router_utils.auto_router_model_naming import validate_complexity_router_config_write + from litellm.router_strategy.auto_router.auto_router_model_naming import validate_complexity_router_config_write from litellm.types.management_endpoints.auto_router_endpoints import ComplexityRouterConfigValidationRequest config: Final = { @@ -1240,7 +1240,7 @@ class TestAutoRouterBenchmarks: def test_the_listed_kinds_match_the_router_types_traffic_can_record(self): from typing import get_args, get_type_hints - from litellm.router_utils.auto_router_model_naming import StrategyRouterKind + from litellm.router_strategy.auto_router.auto_router_model_naming import StrategyRouterKind from litellm.types.utils import StandardLoggingRoutingDecision readonly_router_type: Final = get_type_hints(StandardLoggingRoutingDecision, include_extras=True)["router_type"] diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index fc6749aed0f..437d948f8f5 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -1,6 +1,6 @@ -import inspect import asyncio import contextlib +import inspect import json import socket from collections.abc import Iterator, Mapping @@ -1461,10 +1461,10 @@ class TestDeleteModelClearsRouterRegistry: add_deployment never restores, so an unguarded pop would make it permanently unroutable (the same cross-tenant DoS clear_cache was hardened against). """ + from litellm.proxy.management_endpoints.model_management_endpoints import ModelInfoDelete from litellm.proxy.management_endpoints.model_management_endpoints import ( delete_model as delete_model_endpoint, ) - from litellm.proxy.management_endpoints.model_management_endpoints import ModelInfoDelete model_id = "regular-del-1" admin_user = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) @@ -2739,11 +2739,11 @@ class TestAddAndDeleteModelLifecycle: - Delete same model again → raises (model not found) """ from litellm.proxy.management_endpoints.model_management_endpoints import ( + ModelInfoDelete, add_new_model, - delete_model as delete_model_endpoint, ) from litellm.proxy.management_endpoints.model_management_endpoints import ( - ModelInfoDelete, + delete_model as delete_model_endpoint, ) model_id = "lifecycle-test-model-123" @@ -2833,6 +2833,8 @@ class TestDeleteTeamBYOKModelGhost: async def test_delete_strips_public_name_and_refreshes_cache(self): from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelInfoDelete, + ) + from litellm.proxy.management_endpoints.model_management_endpoints import ( delete_model as delete_model_endpoint, ) @@ -2916,6 +2918,8 @@ class TestDeleteTeamBYOKModelGhost: alias cleanup (delete_team_model_alias), preserving legacy behavior.""" from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelInfoDelete, + ) + from litellm.proxy.management_endpoints.model_management_endpoints import ( delete_model as delete_model_endpoint, ) @@ -2984,6 +2988,8 @@ class TestDeleteTeamBYOKModelGhost: team.models when one replica is deleted but a sibling still backs it.""" from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelInfoDelete, + ) + from litellm.proxy.management_endpoints.model_management_endpoints import ( delete_model as delete_model_endpoint, ) @@ -3065,6 +3071,8 @@ class TestDeleteTeamBYOKModelGhost: deployment still serves it.""" from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelInfoDelete, + ) + from litellm.proxy.management_endpoints.model_management_endpoints import ( delete_model as delete_model_endpoint, ) @@ -3145,6 +3153,8 @@ class TestDeleteTeamBYOKModelGhost: the alias would break routing that works.""" from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelInfoDelete, + ) + from litellm.proxy.management_endpoints.model_management_endpoints import ( delete_model as delete_model_endpoint, ) @@ -3252,6 +3262,8 @@ class TestDeleteModelTeamAuth: async def test_proxy_admin_can_delete_model_when_team_deleted(self): from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelInfoDelete, + ) + from litellm.proxy.management_endpoints.model_management_endpoints import ( delete_model as delete_model_endpoint, ) @@ -3287,6 +3299,8 @@ class TestDeleteModelTeamAuth: """A missing team must never let a non-admin delete the orphan (no fail-open).""" from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelInfoDelete, + ) + from litellm.proxy.management_endpoints.model_management_endpoints import ( delete_model as delete_model_endpoint, ) from litellm.proxy.proxy_server import ProxyException @@ -3322,6 +3336,8 @@ class TestDeleteModelTeamAuth: """The auth check must not add a redundant team query on the live-team path.""" from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelInfoDelete, + ) + from litellm.proxy.management_endpoints.model_management_endpoints import ( delete_model as delete_model_endpoint, ) from litellm.proxy.proxy_server import ProxyException @@ -4392,7 +4408,6 @@ class TestModelInfoCostMapEchoFilter: def test_echoed_cost_map_metadata_is_not_persisted(self): import litellm - from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo @@ -4418,7 +4433,6 @@ class TestModelInfoCostMapEchoFilter: def test_an_edited_value_survives_the_echo_filter(self): import litellm - from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo @@ -4483,7 +4497,6 @@ class TestModelInfoCostMapEchoFilter: def test_metadata_without_a_cost_map_key_is_persisted(self): import litellm - from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo from litellm.types.utils import echoed_cost_map_fields @@ -4513,7 +4526,6 @@ class TestModelInfoCostMapEchoFilter: def test_a_stored_mode_survives_an_echoed_save(self): import litellm - from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo @@ -4536,7 +4548,6 @@ class TestModelInfoCostMapEchoFilter: def test_resetting_an_override_to_the_cost_map_value_removes_it(self): import litellm - from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo @@ -4560,7 +4571,6 @@ class TestModelInfoCostMapEchoFilter: def test_reset_is_recognised_after_the_router_registered_the_override(self, monkeypatch: pytest.MonkeyPatch): import litellm - from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo @@ -4586,7 +4596,6 @@ class TestModelInfoCostMapEchoFilter: from types import MappingProxyType import litellm - from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo @@ -4611,7 +4620,6 @@ class TestModelInfoCostMapEchoFilter: def test_echo_is_compared_against_the_deployments_lookup_not_the_key(self): import litellm - from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo @@ -4653,7 +4661,6 @@ class TestModelInfoCostMapEchoFilter: def test_base_model_wins_over_litellm_params_model_for_the_lookup(self): import litellm - from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo @@ -4681,7 +4688,6 @@ class TestModelInfoCostMapEchoFilter: def test_encrypted_stored_model_is_decrypted_for_the_lookup(self, monkeypatch): import litellm - from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo @@ -6431,7 +6437,7 @@ class TestStrategyRouterWriteValidation: AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY, _auto_router_capability_slot, ) - from litellm.router_utils.auto_router_model_naming import gated_capabilities_of + from litellm.router_strategy.auto_router.auto_router_model_naming import gated_capabilities_of capabilities: Final = gated_capabilities_of(effective_params) @@ -6568,7 +6574,7 @@ class TestStrategyRouterWriteValidation: from fastapi import HTTPException from litellm.proxy.management_endpoints.model_management_endpoints import _auto_router_capability_slot - from litellm.router_utils.auto_router_tuning_baseline import snapshot_tuning_baselines + from litellm.router_strategy.auto_router.auto_router_tuning_baseline import snapshot_tuning_baselines configs = { "_TUNED_A": self._TUNED_A, @@ -6624,7 +6630,7 @@ class TestStrategyRouterWriteValidation: from litellm.proxy import proxy_server from litellm.proxy.management_endpoints.model_management_endpoints import _auto_router_capability_slot - from litellm.router_utils.auto_router_tuning_baseline import snapshot_tuning_baselines + from litellm.router_strategy.auto_router.auto_router_tuning_baseline import snapshot_tuning_baselines fake = self._FakeDb([]) fake.tx_obj.litellm_proxymodeltable.find_many = AsyncMock( @@ -6669,7 +6675,7 @@ class TestStrategyRouterWriteValidation: row id before the slot is entered, so a second tuned router is refused before its DB write.""" from litellm.proxy._types import ProxyException from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model - from litellm.router_utils.auto_router_tuning_baseline import snapshot_tuning_baselines + from litellm.router_strategy.auto_router.auto_router_tuning_baseline import snapshot_tuning_baselines admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) baselines = snapshot_tuning_baselines([self._db_router_row("a", self._TUNED_A)]) @@ -8980,7 +8986,10 @@ class TestTeamMemberAutoRouterWrites: from fastapi import HTTPException from litellm.proxy.management_endpoints.model_management_endpoints import _auto_router_capability_slot - from litellm.proxy.management_helpers.auto_router_permissions import MemberAutoRouterWrite, validate_member_auto_router_config + from litellm.proxy.management_helpers.auto_router_permissions import ( + MemberAutoRouterWrite, + validate_member_auto_router_config, + ) row: Final = self._row() database: Final = self._database(self._team(), row) @@ -10677,6 +10686,8 @@ class TestFederationGateScopesToWhatTheWriteTouches: and the team that owns the model must be able to take it off their page.""" from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelInfoDelete, + ) + from litellm.proxy.management_endpoints.model_management_endpoints import ( delete_model as delete_model_endpoint, ) diff --git a/tests/unit/proxy/management_helpers/test_auto_router_availability.py b/tests/unit/proxy/management_helpers/test_auto_router_availability.py index a88dcd7f66d..0eaae60b4ea 100644 --- a/tests/unit/proxy/management_helpers/test_auto_router_availability.py +++ b/tests/unit/proxy/management_helpers/test_auto_router_availability.py @@ -1,16 +1,15 @@ from collections.abc import Mapping -from typing import Final from types import SimpleNamespace - -from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from typing import Final import pytest +from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.management_helpers.auto_router_availability import ( auto_router_availability, build_auto_router_catalog, ) -from litellm.router_utils.auto_router_tuning_baseline import snapshot_tuning_baselines +from litellm.router_strategy.auto_router.auto_router_tuning_baseline import snapshot_tuning_baselines def deployment( diff --git a/tests/unit/proxy/proxy_server/test_lifecycle.py b/tests/unit/proxy/proxy_server/test_lifecycle.py index 11a44a0ec16..8b1d7c1222a 100644 --- a/tests/unit/proxy/proxy_server/test_lifecycle.py +++ b/tests/unit/proxy/proxy_server/test_lifecycle.py @@ -33,9 +33,9 @@ from pydantic import BaseModel from typing_extensions import TypedDict import litellm.proxy.proxy_server as ps +from litellm.proxy.management_helpers.audit_logs import track_audit_task from litellm.proxy.proxy_server import ( ProxyStartupEvent, - initialize_shared_aiohttp_session, _resolve_pydantic_type, _resolve_typed_dict_type, cleanup_router_config_variables, @@ -43,6 +43,7 @@ from litellm.proxy.proxy_server import ( get_litellm_model_info, initialize, initialize_from_worker_config, + initialize_shared_aiohttp_session, load_from_azure_key_vault, proxy_shutdown_event, proxy_startup_event, @@ -50,8 +51,6 @@ from litellm.proxy.proxy_server import ( save_worker_config, ) -from litellm.proxy.management_helpers.audit_logs import track_audit_task - from .conftest import normalize # --------------------------------------------------------------------------- @@ -1165,7 +1164,7 @@ def test_proxy_startup_event_warns_for_global_budget_without_database(): @pytest.mark.asyncio async def test_tuning_baseline_v3_is_created_alongside_the_legacy_row(): - from litellm.router_utils.auto_router_tuning_baseline import DEFAULT_TUNING_FINGERPRINT + from litellm.router_strategy.auto_router.auto_router_tuning_baseline import DEFAULT_TUNING_FINGERPRINT prisma_client = MagicMock() prisma_client.db.litellm_config.find_unique = AsyncMock(return_value=None) @@ -1186,7 +1185,10 @@ async def test_tuning_baseline_v3_is_created_alongside_the_legacy_row(): @pytest.mark.asyncio async def test_scorer_baseline_upgrade_preserves_existing_routers_and_is_not_refreshed_on_restart(): - from litellm.router_utils.auto_router_tuning_baseline import mutable_tuned_identities, snapshot_tuning_baselines + from litellm.router_strategy.auto_router.auto_router_tuning_baseline import ( + mutable_tuned_identities, + snapshot_tuning_baselines, + ) deployments = [ { diff --git a/tests/unit/router_utils/test_auto_router_model_naming.py b/tests/unit/router_strategy/test_auto_router_model_naming.py similarity index 99% rename from tests/unit/router_utils/test_auto_router_model_naming.py rename to tests/unit/router_strategy/test_auto_router_model_naming.py index 1cfa3aa5c91..44fcfdaf5be 100644 --- a/tests/unit/router_utils/test_auto_router_model_naming.py +++ b/tests/unit/router_strategy/test_auto_router_model_naming.py @@ -3,9 +3,7 @@ from typing import Final import pytest -from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets -from litellm.router_strategy.complexity_router.jev_classifier import DEFAULT_JEV_INSTRUCTIONS -from litellm.router_utils.auto_router_model_naming import ( +from litellm.router_strategy.auto_router.auto_router_model_naming import ( GATED_AUTO_ROUTER_CAPABILITIES, capability_limit_violation, carries_complexity_router_settings, @@ -18,6 +16,8 @@ from litellm.router_utils.auto_router_model_naming import ( validate_complexity_router_config_write, validate_strategy_router_model_write, ) +from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets +from litellm.router_strategy.complexity_router.jev_classifier import DEFAULT_JEV_INSTRUCTIONS COMPLEXITY_FIELDS = frozenset({"complexity_router_config"}) SEMANTIC_FIELDS = frozenset({"auto_router_config", "auto_router_default_model", "auto_router_embedding_model"}) @@ -596,7 +596,7 @@ def test_custom_classifier_prompt_capability(config: Mapping[str, object], expec ], ) def test_is_complexity_router_model(model: str | None, expected: bool) -> None: - from litellm.router_utils.auto_router_model_naming import is_complexity_router_model + from litellm.router_strategy.auto_router.auto_router_model_naming import is_complexity_router_model assert is_complexity_router_model(model) is expected diff --git a/tests/unit/router_utils/test_auto_router_tuning_baseline.py b/tests/unit/router_strategy/test_auto_router_tuning_baseline.py similarity index 99% rename from tests/unit/router_utils/test_auto_router_tuning_baseline.py rename to tests/unit/router_strategy/test_auto_router_tuning_baseline.py index d0d98ce7042..4ee219a7784 100644 --- a/tests/unit/router_utils/test_auto_router_tuning_baseline.py +++ b/tests/unit/router_strategy/test_auto_router_tuning_baseline.py @@ -7,8 +7,7 @@ from typing import Final import pytest -from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig -from litellm.router_utils.auto_router_tuning_baseline import ( +from litellm.router_strategy.auto_router.auto_router_tuning_baseline import ( DEFAULT_TUNING_FINGERPRINT, HEURISTIC_V1_TUNING_FIELDS, heuristic_v1_router_fingerprint, @@ -19,6 +18,7 @@ from litellm.router_utils.auto_router_tuning_baseline import ( tuning_limit_violation, tuning_quota_violation, ) +from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig _TIERS = {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"} _ALT_TIERS = {**_TIERS, "COMPLEX": "other-strong"} diff --git a/tests/unit/router_utils/test_baseline_request.py b/tests/unit/router_strategy/test_baseline_request.py similarity index 95% rename from tests/unit/router_utils/test_baseline_request.py rename to tests/unit/router_strategy/test_baseline_request.py index 37d21597752..eb785a41d63 100644 --- a/tests/unit/router_utils/test_baseline_request.py +++ b/tests/unit/router_strategy/test_baseline_request.py @@ -1,6 +1,6 @@ from typing import Final -from litellm.router_utils.baseline_request import baseline_request, capture_baseline_parameters +from litellm.router_strategy.auto_router.baseline_request import baseline_request, capture_baseline_parameters def test_baseline_snapshot_owns_nested_caller_settings_and_overrides_routed_settings() -> None: diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index ca39c7f5c8f..018f93779a6 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -24,14 +24,6 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm import Router -from litellm.integrations.custom_logger import CustomLogger -from litellm.types.utils import StandardLoggingRoutingDecision -from litellm.router_utils.auto_router_model_naming import ( - CUSTOMIZATION_CAPABILITY, - GATED_AUTO_ROUTER_CAPABILITIES, - HEURISTIC_V2_CAPABILITY, - count_capability_routers, -) from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache @@ -42,7 +34,19 @@ from litellm.constants import ( RETURN_RAW_MODEL_NAME_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY, ) +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.router import as_output_cap, stamped_client_ceiling +from litellm.router_strategy.auto_router.auto_router_model_naming import ( + CUSTOMIZATION_CAPABILITY, + GATED_AUTO_ROUTER_CAPABILITIES, + HEURISTIC_V2_CAPABILITY, + count_capability_routers, +) +from litellm.router_strategy.complexity_router.capability_classifier import ( + CAPABILITY_CLASSIFIER_SYSTEM_PROMPT, + CapabilityClassifierVerdict, +) from litellm.router_strategy.complexity_router.complexity_router import ( _CLASSIFICATION_CURRENT_MESSAGE_ONLY, _CLASSIFICATION_WITH_CONVERSATION, @@ -58,18 +62,14 @@ from litellm.router_strategy.complexity_router.complexity_router import ( classification_system_prompt, custom_tier_classification_prompt, ) -from litellm.router_strategy.complexity_router.capability_classifier import ( - CAPABILITY_CLASSIFIER_SYSTEM_PROMPT, - CapabilityClassifierVerdict, -) from litellm.router_strategy.complexity_router.config import ( - CapabilityCalibrationConfig, - CapabilityClassifierConfig, DEFAULT_CLASSIFICATION_RUBRIC, DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE, DEFAULT_COMPLEXITY_CONFIG, DEFAULT_TECHNICAL_KEYWORDS, TIER_SEVERITY_ORDER, + CapabilityCalibrationConfig, + CapabilityClassifierConfig, ClassificationRubric, ClassifierLLMConfig, ComplexityRouterConfig, @@ -83,10 +83,12 @@ from litellm.router_strategy.complexity_router.jev_classifier import ( JevUsage, ) from litellm.router_strategy.complexity_router.llm_v2 import LLM_V2_PROMPT_VERSION +from litellm.router_strategy.complexity_router.request_models import RequestComplexityRouterConfig from litellm.router_strategy.complexity_router.tier_predictor import ( TierGlobalStatistic, TrainedTierArtifact, ) +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.router import ( Deployment, LiteLLM_Params, @@ -94,10 +96,7 @@ from litellm.types.router import ( RouterErrors, TaggedPreRoutingStrategy, ) -from litellm.types.llms.openai import ResponsesAPIResponse -from litellm.router_strategy.complexity_router.request_models import RequestComplexityRouterConfig -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler - +from litellm.types.utils import StandardLoggingRoutingDecision requires_semantic_router = pytest.mark.skipif( sys.version_info >= (3, 14), reason="The semantic-router extra excludes Python 3.14" @@ -16299,7 +16298,6 @@ class TestTierHealthFailover: tests can prove the probe passes a kwargs copy and forwards the prompt arguments. """ import litellm as litellm_module - from litellm.types.router import RouterRateLimitError probed_kwargs = [] diff --git a/tests/unit/router_strategy/test_llm_v2.py b/tests/unit/router_strategy/test_llm_v2.py index 3fd2e8808e7..8c1e9e48661 100644 --- a/tests/unit/router_strategy/test_llm_v2.py +++ b/tests/unit/router_strategy/test_llm_v2.py @@ -4,11 +4,12 @@ from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest -import litellm from pydantic import ValidationError +import litellm from litellm import ModelResponse, Router from litellm.caching.dual_cache import DualCache +from litellm.router_strategy.auto_router.auto_router_model_naming import strategy_router_dependencies from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig, ComplexityTier from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets @@ -21,7 +22,6 @@ from litellm.router_strategy.complexity_router.llm_v2 import ( LLMV2Verdict, llm_v2_response_format, ) -from litellm.router_utils.auto_router_model_naming import strategy_router_dependencies from litellm.types.llms.openai import ResponsesAPIResponse diff --git a/tests/unit/test_private_usage_aliases.py b/tests/unit/test_private_usage_aliases.py index 1d9c2e87100..d0c5c8247d3 100644 --- a/tests/unit/test_private_usage_aliases.py +++ b/tests/unit/test_private_usage_aliases.py @@ -1969,7 +1969,7 @@ ALIAS_CASES: Final = ( ("litellm.router_utils.add_retry_fallback_headers", "HiddenParamsAsyncIteratorWrapper", "_inner", "inner", True), ("litellm.router_utils.add_retry_fallback_headers", "", "_HiddenParamsHost", "HiddenParamsHost", False), ( - "litellm.router_utils.batch_utils", + "litellm.litellm_core_utils.batch_utils", "", "_get_router_metadata_variable_name", "get_router_metadata_variable_name", diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index af8e7f53462..513136ae150 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -4,20 +4,21 @@ import gc import json import os import weakref -from datetime import datetime, timezone from collections.abc import Iterator, Mapping +from datetime import datetime, timezone from types import MethodType from typing import Final from unittest.mock import AsyncMock, MagicMock, patch -import litellm import pytest + +import litellm from litellm import Router from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY -from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook from litellm.types.litellm_params import RoutingStrategyName @@ -801,10 +802,11 @@ async def test_deployment_callback_on_failure(model_list): def test_deployment_callback_respects_cooldown_time(model_list): """Ensure per-model cooldown_time is honored even when exception headers are present.""" - import httpx import time from unittest.mock import patch + import httpx + router = Router(model_list=model_list) class FakeException(Exception): @@ -1811,8 +1813,8 @@ def test_get_allowed_fails_from_policy(model_list, exception_type, exception_nam def test_initialize_alerting(model_list): """Test if the 'initialize_alerting' function is working correctly""" - from litellm.types.router import AlertingConfig from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + from litellm.types.router import AlertingConfig router = Router( model_list=model_list, alerting_config=AlertingConfig(webhook_url="test") @@ -2245,6 +2247,7 @@ def test_deployments_by_pattern(model_list): ) def test_pattern_match_deployment_set_model_name(user_request_model, model_name, litellm_model, expected_model): from re import Match + from litellm.router_utils.pattern_match_deployments import PatternMatchRouter pattern_router = PatternMatchRouter() @@ -2702,7 +2705,7 @@ def test_handle_clientside_credential_metadata_loading(model_list, function_name ) def test_handle_clientside_credential_metadata_variable_name(model_list, function_name, metadata_key): """Test that _handle_clientside_credential uses the correct metadata variable name based on function name""" - from litellm.router_utils.batch_utils import get_router_metadata_variable_name + from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name router = Router(model_list=model_list) diff --git a/tests/unit/test_router/test_router_surface_check.py b/tests/unit/test_router/test_router_surface_check.py new file mode 100644 index 00000000000..a97fd029380 --- /dev/null +++ b/tests/unit/test_router/test_router_surface_check.py @@ -0,0 +1,58 @@ +""" +The router surface fence (tests/code_coverage_tests/router_surface_check.py) must see every way +outside code reaches a Router member, or new dependencies slip past the shrink-only allowlist. +""" + +import ast +import importlib.util +import pathlib + +import pytest + +_CHECK = pathlib.Path(__file__).parents[2] / "code_coverage_tests" / "router_surface_check.py" + + +@pytest.fixture(scope="module") +def surface_check(): + spec = importlib.util.spec_from_file_location("router_surface_check", _CHECK) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _uses(surface_check, src: str) -> set: + return surface_check.member_uses(ast.parse(src), surface_check.router_members()) + + +def test_instance_and_dynamic_access_are_uses(surface_check): + assert _uses(surface_check, "llm_router.get_model_list()\ngetattr(router, 'model_list')") == { + "get_model_list", + "model_list", + } + + +def test_class_level_calls_are_uses(surface_check): + assert _uses(surface_check, "litellm.Router.get_valid_args()") == {"get_valid_args"} + assert _uses(surface_check, "from litellm import Router\nRouter.get_valid_args()") == {"get_valid_args"} + + +def test_container_file_endpoints_registered_by_name_are_members(surface_check): + members = surface_check.router_members() + assert {"aupload_container_file", "upload_container_file", "alist_container_files"} <= members + assert _uses(surface_check, "await llm_router.aupload_container_file(file=f)") == {"aupload_container_file"} + + +def test_unlisted_class_level_use_fails_the_fence(surface_check): + found = { + "members": {"outside": {"get_valid_args": {"litellm/new_module.py"}}, "router_family": {}}, + "router_utils_imports": {}, + } + allow = {"members_outside": [], "members_router_family": [], "router_utils_imports_outside": []} + errors = surface_check.compare(allow, found) + assert len(errors) == 1 + assert "'get_valid_args' in litellm/new_module.py" in errors[0] + + +def test_unrelated_receivers_are_not_uses(surface_check): + assert _uses(surface_check, "client.get_model_list()\nRouterConfig.model_list") == set() From 49ccce7813725cc87b64d1b0f1b097dca5c681af Mon Sep 17 00:00:00 2001 From: paveln83 <349044532+paveln83@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:12:07 -0700 Subject: [PATCH 27/27] test(extraction): audit #47473 (#48898) * test(extraction): audit #47473 * test(extraction): check over-budget extraction never reaches Cloudflare --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../management/test_model_health_check.py | 37 ++ ...test_cloudflare_clairic_extraction_wire.py | 418 ++++++++++++++++++ 2 files changed, 455 insertions(+) create mode 100644 tests/integration/providers/test_cloudflare_clairic_extraction_wire.py diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 57a05daca94..992c1e8e875 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -26,6 +26,20 @@ _DECISIONS_PROBE_REPLY: Final = JsonResponse( "usage": {"input_tokens": 10, "output_tokens": 1}, }, ) +_EXTRACTION_PROBE_REPLY: Final = JsonResponse( + content_type="application/json", + body={ + "result": { + "model": "clairic", + "answers": {"reachable": {"type": "text", "value": "health", "confidence": 0.9}}, + "truncated": False, + "usage": {"input_tokens": 10, "output_tokens": 0}, + }, + "success": True, + "errors": [], + "messages": [], + }, +) _CONFIGURED_PROBE_REPLY: Final = JsonResponse( content_type="application/json", body={ @@ -287,6 +301,29 @@ def test_decisions_mode_health_check_resolves_the_mode_from_the_cost_map_and_sen ] +def test_extraction_mode_health_check_resolves_the_mode_from_the_cost_map_and_sends_the_extraction_probe( + gateway: Gateway, +) -> None: + with gateway.scenario() as scenario: + handle: Final = register_scenario(f"health-extraction-{uuid.uuid4().hex[:12]}", _EXTRACTION_PROBE_REPLY) + scenario.cleanups.callback(delete_scenario, handle) + model: Final = scenario.model( + model="cloudflare/@cf/cloudflare/clairic", api_base=handle.api_base(), api_key="synthetic-cloudflare-key" + ) + report: Final = _health_report(gateway, model) + assert (report["healthy_count"], report["unhealthy_count"]) == (1, 0), report + assert _probes_sent_to(gateway, handle) == [ + ( + f"/{handle.scenario_id}/ai/run/@cf/cloudflare/clairic", + { + "model": "clairic", + "state": os.environ.get("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"), + "fields": {"reachable": {"type": "text", "instructions": "Any word from the text"}}, + }, + ) + ] + + @pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation")) def test_decisions_mode_health_check_sends_the_configured_state_and_questions(gateway: Gateway, mode: str) -> None: with gateway.scenario() as scenario: diff --git a/tests/integration/providers/test_cloudflare_clairic_extraction_wire.py b/tests/integration/providers/test_cloudflare_clairic_extraction_wire.py new file mode 100644 index 00000000000..7b2a48c9676 --- /dev/null +++ b/tests/integration/providers/test_cloudflare_clairic_extraction_wire.py @@ -0,0 +1,418 @@ +import json +import math +import re +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal + +import httpx +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, eventually, object_value +from integration._support.database import read_rows +from integration._support.process import OwnedProxy, WorkerHealthcheck, graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +Route = Literal["/v1/extraction", "/extraction"] + +_API_KEY: Final = "synthetic-cloudflare-key" +_CLAIRIC: Final = "cloudflare/@cf/cloudflare/clairic" +_CLAIRIC_PATH: Final = "/ai/run/@cf/cloudflare/clairic" +_INPUT_TOKENS: Final = 412 +_EMAIL: Final = "billing@acmetools.example" +_SCENARIO: Final = re.compile(r"scenario-([a-z-]+)") +_JSON_ROWS: Final = TypeAdapter(list[dict[str, JsonValue]]) +_FIELDS: Final[dict[str, JsonValue]] = { + "vendor": {"type": "text", "instructions": "Who sent the invoice?"}, + "total": {"type": "number", "unit": "USD"}, +} +_SPEND_QUERY: Final = ( + 'SELECT spend, status, call_type, model_group, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" ' + "WHERE request_id = %s" +) +_OWNED_TIMEOUT: Final = int(2 * graceful_stop_seconds() + 120) +_REQUEST_GUARDRAIL: Final = "extraction-mask-email-in" +_RESPONSE_GUARDRAIL: Final = "extraction-mask-email-out" +_OWNED_MODEL: Final = "invoice-reader" + + +def _state(marker: str, scenario: str = "plain") -> str: + return f"Invoice from Acme Tools, total $174. scenario-{scenario} marker-{marker}" + + +def _vendor_span(state: str) -> list[JsonValue]: + start: Final = state.index("Acme Tools") + return [start, start + len("Acme Tools")] + + +def _email_span(state: str) -> list[JsonValue]: + start: Final = state.index(_EMAIL) + return [start, start + len(_EMAIL)] + + +def _answers(state: str) -> dict[str, JsonValue]: + return { + "vendor": {"type": "text", "value": "Acme Tools", "confidence": 0.97, "span": _vendor_span(state)}, + "total": {"type": "number", "value": 174, "confidence": 0.95}, + } + + +def _envelope(result: Mapping[str, JsonValue]) -> Reply: + return Reply(body=json.dumps({"result": result, "success": True, "errors": [], "messages": []}).encode()) + + +def _clairic_reply(request: Request) -> Reply: + state: Final = JSON_OBJECT.validate_json(request.body)["state"] + assert isinstance(state, str), request.body + return _envelope( + { + "model": "clairic", + "answers": _answers(state), + "truncated": False, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0}, + } + ) + + +def _expected_response(model: str, state: str) -> dict[str, JsonValue]: + return { + "model": model, + "answers": _answers(state), + "truncated": False, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0}, + } + + +def _upstream_body(request: Request) -> dict[str, JsonValue]: + assert (request.method, request.target) == ("POST", _CLAIRIC_PATH), (request.method, request.target) + assert request.headers["authorization"] == f"Bearer {_API_KEY}", request.headers + return JSON_OBJECT.validate_json(request.body) + + +def _body(model: str, marker: str, **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "state": _state(marker), "fields": _FIELDS, **extra} + + +def _extract( + gateway: Gateway, + model: str, + marker: str, + *, + route: Route = "/v1/extraction", + key: str | None = None, + **extra: JsonValue, +) -> httpx.Response: + return gateway.request("POST", route, _body(model, marker, **extra), key=key) + + +def _number(value: JsonValue) -> float: + assert isinstance(value, (int, float)) and not isinstance(value, bool), value + return float(value) + + +def _expected_spend() -> float: + cost_map: Final = JSON_OBJECT.validate_json(Path("model_prices_and_context_window.json").read_bytes()) + priced: Final = cost_map.get(_CLAIRIC) + assert priced is not None, f"{_CLAIRIC} has no cost map entry" + return _INPUT_TOKENS * _number(object_value(priced)["input_cost_per_token"]) + + +def _spend_row(call_id: str) -> dict[str, JsonValue]: + rows: Final = eventually(lambda: read_rows(_SPEND_QUERY, (call_id,)), lambda found: len(found) == 1, seconds=70) + return rows[0] + + +def _assert_billed(gateway: Gateway, response: httpx.Response, model: str) -> None: + expected: Final = _expected_spend() + assert math.isclose(float(response.headers["x-litellm-response-cost"]), expected, rel_tol=1e-9), response.headers + call_id: Final = response.headers["x-litellm-call-id"] + row: Final = _spend_row(call_id) + assert (row["status"], row["call_type"], row["model_group"], row["prompt_tokens"], row["completion_tokens"]) == ( + "success", + "aextraction", + model, + _INPUT_TOKENS, + 0, + ), row + assert math.isclose(_number(row["spend"]), expected, rel_tol=1e-9), row + listed: Final = gateway.request("GET", "/spend/logs", params={"request_id": call_id}) + assert listed.status_code == 200, listed.text + (reported,) = _JSON_ROWS.validate_json(listed.content) + assert (reported["request_id"], reported["model_group"]) == (call_id, model), reported + assert math.isclose(_number(reported["spend"]), expected, rel_tol=1e-9), reported + + +@pytest.mark.parametrize("route", ("/v1/extraction", "/extraction")) +def test_extraction_reaches_clairic_on_both_routes_and_is_billed(gateway: Gateway, route: Route) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clairic_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY) + response: Final = _extract(gateway, model, marker, route=route, metadata={"tags": [f"audit-{marker}"]}) + assert response.status_code == 200, response.text + assert response.json() == _expected_response(model, _state(marker)) + (request,) = wire.drain() + assert _upstream_body(request) == {"model": "clairic", "state": _state(marker), "fields": _FIELDS} + _assert_billed(gateway, response, model) + + +_INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( + ("unknown_type", {"fields": {"vendor": {"type": "paragraph"}}}), + ("list_without_items", {"fields": {"items": {"type": "list"}}}), + ("remote_image", {"images": ["https://images.example/receipt.png"]}), +) + + +@pytest.mark.parametrize(("label", "change"), _INVALID_BODIES, ids=[label for label, _ in _INVALID_BODIES]) +def test_invalid_extraction_bodies_are_refused_before_any_upstream_call( + gateway: Gateway, label: str, change: dict[str, JsonValue] +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clairic_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request("POST", "/v1/extraction", {**_body(model, marker), **change}) + assert response.status_code == 400, (label, response.text) + assert "Invalid Extraction request" in response.text, (label, response.text) + assert wire.drain() == (), label + + +@pytest.mark.parametrize(("param", "value"), (("temperature", 0.2), ("stream", True)), ids=("temperature", "stream")) +def test_unsupported_top_level_params_are_refused_unless_the_deployment_drops_params( + gateway: Gateway, param: str, value: JsonValue +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clairic_reply) as wire, gateway.scenario() as scenario: + strict: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY) + dropping: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY, drop_params=True) + refused: Final = _extract(gateway, strict, marker, **{param: value}) + assert refused.status_code == 400, refused.text + assert param in refused.text, refused.text + assert wire.drain() == () + dropped: Final = _extract(gateway, dropping, marker, **{param: value}) + assert dropped.status_code == 200, dropped.text + assert dropped.headers["content-type"].startswith("application/json"), dropped.headers + assert dropped.json() == _expected_response(dropping, _state(marker)) + (request,) = wire.drain() + assert _upstream_body(request) == {"model": "clairic", "state": _state(marker), "fields": _FIELDS} + _assert_billed(gateway, dropped, dropping) + + +def test_key_checks_on_extraction_match_chat(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_clairic_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY) + anonymous: Final = gateway.client.post("/v1/extraction", json=_body(model, marker)) + assert anonymous.status_code == 401, anonymous.text + restricted: Final = scenario.key(models=[f"other-{uuid.uuid4().hex}"]) + refused: Final = _extract(gateway, model, marker, key=restricted) + chat_refused: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": marker}]}, + key=restricted, + ) + assert 400 <= refused.status_code < 500, refused.text + assert refused.status_code == chat_refused.status_code, (refused.text, chat_refused.text) + assert wire.drain() == () + allowed: Final = scenario.key(models=[model], max_budget=1e-06) + first: Final = _extract(gateway, model, marker, key=allowed) + assert first.status_code == 200, first.text + assert len(wire.drain()) == 1 + blocked, upstream_calls = eventually( + lambda: (_extract(gateway, model, uuid.uuid4().hex, key=allowed), len(wire.drain())), + lambda answer: answer[0].status_code != 200, + seconds=70, + ) + chat_blocked: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": marker}]}, + key=allowed, + ) + assert 400 <= blocked.status_code < 500, blocked.text + assert blocked.status_code == chat_blocked.status_code, (blocked.text, chat_blocked.text) + assert upstream_calls == 0 + assert wire.drain() == () + + +def test_a_small_key_tpm_limit_serves_extraction_and_counts_its_input_tokens(gateway: Gateway) -> None: + with wire_server(_clairic_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY) + key: Final = scenario.key(models=[model], tpm_limit=100) + first: Final = _extract(gateway, model, uuid.uuid4().hex, key=key) + assert first.status_code == 200, first.text + assert len(wire.drain()) == 1 + limited, upstream_calls = eventually( + lambda: (_extract(gateway, model, uuid.uuid4().hex, key=key), len(wire.drain())), + lambda answer: answer[0].status_code != 200, + seconds=70, + ) + assert limited.status_code == 429, limited.text + assert upstream_calls == 0 + + +def test_cloudflare_errors_keep_their_status_and_log_an_unbilled_failure(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + refusal: Final = json.dumps( + { + "result": None, + "success": False, + "errors": [{"code": 5006, "message": f"min above max {marker}"}], + "messages": [], + } + ).encode() + with wire_server(lambda _: Reply(status=422, body=refusal)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY) + response: Final = _extract(gateway, model, marker) + assert response.status_code == 422, response.text + assert f"min above max {marker}" in response.text, response.text + assert len(wire.drain()) >= 1 + row: Final = _spend_row(response.headers["x-litellm-call-id"]) + assert (row["status"], row["call_type"], row["model_group"], _number(row["spend"])) == ( + "failure", + "aextraction", + model, + 0.0, + ), row + + +def _owned_reply(request: Request) -> Reply: + body: Final = JSON_OBJECT.validate_json(request.body) + state: Final = body["state"] + assert isinstance(state, str), request.body + scenario: Final = _SCENARIO.search(state) + assert scenario is not None, state + if scenario.group(1) != "mask-out": + return _clairic_reply(request) + return _envelope( + { + "model": "clairic", + "answers": { + "vendor": {"type": "text", "value": "Acme Tools", "confidence": 0.97, "span": _vendor_span(state)}, + "contact_email": {"type": "text", "value": _EMAIL, "confidence": 0.9, "span": _email_span(state)}, + }, + "truncated": False, + "state_text": state, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0}, + } + ) + + +def _owned_config(upstream_url: str) -> dict[str, JsonValue]: + base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + email_filter: Final[list[JsonValue]] = [{"pattern_type": "prebuilt", "pattern_name": "email", "action": "MASK"}] + return { + **base, + "guardrails": [ + { + "guardrail_name": _REQUEST_GUARDRAIL, + "litellm_params": {"guardrail": "litellm_content_filter", "mode": "pre_call", "patterns": email_filter}, + }, + { + "guardrail_name": _RESPONSE_GUARDRAIL, + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "post_call", + "patterns": email_filter, + }, + }, + ], + "model_list": [ + { + "model_name": _OWNED_MODEL, + "litellm_params": {"model": _CLAIRIC, "api_base": upstream_url, "api_key": _API_KEY}, + } + ], + } + + +@dataclass(frozen=True, slots=True) +class Rig: + proxy: OwnedProxy + upstream: Wire + + +@pytest.fixture(scope="module") +def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + with wire_server(_owned_reply) as upstream: + directory: Final = tmp_path_factory.mktemp("clairic-extraction-owned") + config_path: Final = directory / "clairic-extraction.yaml" + config_path.write_text(yaml.safe_dump(_owned_config(upstream.url))) + with owned_proxy_process( + session_gateway, + directory, + {}, + config=config_path, + workers=2, + extra_arguments=("--max_tokens", "256", "--temperature", "0.2"), + healthcheck=WorkerHealthcheck.LENIENT, + ) as proxy: + yield Rig(proxy, upstream) + + +def _owned_extract(rig: Rig, state: str, **extra: JsonValue) -> httpx.Response: + return rig.proxy.gateway.request( + "POST", "/v1/extraction", {"model": _OWNED_MODEL, "state": state, "fields": _FIELDS, **extra} + ) + + +def _applied_guardrails(response: httpx.Response) -> frozenset[str]: + return frozenset(name.strip() for name in response.headers.get("x-litellm-applied-guardrails", "").split(",")) + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_cli_generation_defaults_never_reach_an_extraction_call(rig: Rig) -> None: + state: Final = _state(uuid.uuid4().hex) + response: Final = _owned_extract(rig, state) + assert response.status_code == 200, response.text + assert response.json() == _expected_response(_OWNED_MODEL, state) + (request,) = rig.upstream.drain() + assert _upstream_body(request) == {"model": "clairic", "state": state, "fields": _FIELDS} + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_a_request_guardrail_masks_the_state_before_clairic_and_drops_spans_into_it(rig: Rig) -> None: + state: Final = f"{_state(uuid.uuid4().hex, 'mask-in')} Questions go to {_EMAIL}." + response: Final = _owned_extract(rig, state, guardrails=[_REQUEST_GUARDRAIL]) + assert response.status_code == 200, response.text + assert _REQUEST_GUARDRAIL in _applied_guardrails(response), response.headers + (request,) = rig.upstream.drain() + sent: Final = _upstream_body(request) + sent_state: Final = sent["state"] + assert isinstance(sent_state, str) and _EMAIL not in sent_state, sent + assert sent_state.startswith(state.removesuffix(f"{_EMAIL}.")) and "REDACTED" in sent_state, sent + assert sent == {"model": "clairic", "state": sent_state, "fields": _FIELDS} + answers: Final = object_value(response.json()["answers"]) + assert object_value(answers["vendor"]) == {"type": "text", "value": "Acme Tools", "confidence": 0.97}, answers + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_a_response_guardrail_masks_text_answers_and_state_text(rig: Rig) -> None: + state: Final = f"{_state(uuid.uuid4().hex, 'mask-out')} Questions go to {_EMAIL}." + fields: Final[dict[str, JsonValue]] = {**_FIELDS, "contact_email": {"type": "text"}} + response: Final = rig.proxy.gateway.request( + "POST", + "/v1/extraction", + { + "model": _OWNED_MODEL, + "state": state, + "fields": fields, + "guardrails": [_RESPONSE_GUARDRAIL], + "options": {"return_state_text": True}, + }, + ) + assert response.status_code == 200, response.text + assert _RESPONSE_GUARDRAIL in _applied_guardrails(response), response.headers + (request,) = rig.upstream.drain() + assert _upstream_body(request)["state"] == state + body: Final = response.json() + state_text: Final = body["state_text"] + assert isinstance(state_text, str) and _EMAIL not in state_text and "REDACTED" in state_text, body + answers: Final = object_value(body["answers"]) + contact: Final = object_value(answers["contact_email"]) + assert "span" not in contact, answers + masked: Final = contact["value"] + assert isinstance(masked, str) and _EMAIL not in masked and "REDACTED" in masked, answers + assert object_value(answers["vendor"]) == {"type": "text", "value": "Acme Tools", "confidence": 0.97}, answers