From 6b591c34f141d4977078da2e7d67ad320d898f13 Mon Sep 17 00:00:00 2001 From: KunalG67 Date: Mon, 27 Apr 2026 17:15:11 +0530 Subject: [PATCH 01/85] fix(ovhcloud): migrate reasoning_content->reasoning and duration->seconds fields OVHCloud is deprecating two response fields on 2026-05-11: - reasoning_content replaced by reasoning (LLM reasoning models) - duration replaced by seconds (Speech-to-Text models) Adds backward-compatible support for both field names during the transition window, preferring the new field when present and falling back to the legacy field. Fixes #26586 --- .../audio_transcription/transformation.py | 8 ++ litellm/llms/ovhcloud/chat/transformation.py | 14 +++- ...loud_audio_transcription_transformation.py | 43 +++++++++++ .../test_ovhcloud_chat_transformation.py | 73 +++++++++++++++++++ 4 files changed, 134 insertions(+), 4 deletions(-) diff --git a/litellm/llms/ovhcloud/audio_transcription/transformation.py b/litellm/llms/ovhcloud/audio_transcription/transformation.py index 7ff6dc986be..e3c8308d507 100644 --- a/litellm/llms/ovhcloud/audio_transcription/transformation.py +++ b/litellm/llms/ovhcloud/audio_transcription/transformation.py @@ -156,5 +156,13 @@ class OVHCloudAudioTranscriptionConfig(BaseAudioTranscriptionConfig): text = response_json.get("text") or response_json.get("transcript") or "" response = TranscriptionResponse(text=text) + # OVHCloud field migration (deadline: 2026-05-11): + # `duration` is replaced by `seconds` in STT responses. + # Prefer `seconds`, fall back to `duration`, normalize to `duration` + # so downstream consumers see a consistent key. + duration = response_json.get("seconds") or response_json.get("duration") + if duration is not None: + response_json["duration"] = duration + response._hidden_params = response_json return response diff --git a/litellm/llms/ovhcloud/chat/transformation.py b/litellm/llms/ovhcloud/chat/transformation.py index ae9271ddb16..4100c548f2c 100644 --- a/litellm/llms/ovhcloud/chat/transformation.py +++ b/litellm/llms/ovhcloud/chat/transformation.py @@ -98,10 +98,16 @@ class OVHCloudChatCompletionStreamingHandler(BaseModelResponseIterator): new_choices = [] for choice in chunk["choices"]: - if "delta" in choice and "reasoning" in choice["delta"]: - choice["delta"]["reasoning_content"] = choice["delta"].get( - "reasoning" - ) + if "delta" in choice: + delta = choice["delta"] + # OVHCloud field migration (deadline: 2026-05-11): + # `reasoning_content` is replaced by `reasoning`. + # Normalise to `reasoning_content` so downstream consumers + # see a consistent key during the transition window. + reasoning_new = delta.get("reasoning") + reasoning_legacy = delta.get("reasoning_content") + if reasoning_new is not None and reasoning_legacy is None: + delta["reasoning_content"] = reasoning_new new_choices.append(choice) return ModelResponseStream( diff --git a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py index 8cc46dc98d0..e9abf50ba75 100644 --- a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py +++ b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py @@ -54,3 +54,46 @@ def test_ovhcloud_audio_transcription_config_installed(): assert config is not None assert isinstance(config, BaseAudioTranscriptionConfig) + + + +class TestOVHCloudDurationFieldMigration: + """Tests for OVHCloud duration -> seconds field migration.""" + + def test_seconds_field_mapped_to_duration(self): + """New `seconds` field should be normalized to `duration`.""" + from litellm.llms.ovhcloud.audio_transcription.transformation import ( + OVHCloudAudioTranscriptionConfig, + ) + from unittest.mock import MagicMock + + config = OVHCloudAudioTranscriptionConfig() + mock_response = MagicMock() + mock_response.json.return_value = { + "text": "Hello world", + "seconds": 3.14, + } + + result = config.transform_audio_transcription_response(mock_response) + + assert result.text == "Hello world" + assert result._hidden_params["duration"] == 3.14 + + def test_legacy_duration_field_still_works(self): + """Legacy `duration` field should still be accepted.""" + from litellm.llms.ovhcloud.audio_transcription.transformation import ( + OVHCloudAudioTranscriptionConfig, + ) + from unittest.mock import MagicMock + + config = OVHCloudAudioTranscriptionConfig() + mock_response = MagicMock() + mock_response.json.return_value = { + "text": "Hello world", + "duration": 2.71, + } + + result = config.transform_audio_transcription_response(mock_response) + + assert result.text == "Hello world" + assert result._hidden_params["duration"] == 2.71 \ No newline at end of file diff --git a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py index a1b3b31f786..88ce3b4c296 100644 --- a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py +++ b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py @@ -292,3 +292,76 @@ def test_ovhcloud_with_custom_base_url(): if __name__ == "__main__": pytest.main([__file__, "-v"]) + + +class TestOVHCloudReasoningFieldMigration: + """Tests for OVHCloud reasoning_content -> reasoning field migration.""" + + def test_streaming_new_reasoning_field(self): + """New `reasoning` field should be mapped to `reasoning_content`.""" + handler = OVHCloudChatCompletionStreamingHandler( + streaming_response=iter([]), + sync_stream=True, + ) + chunk = { + "id": "test-id", + "created": 1234567890, + "model": "test-model", + "choices": [ + { + "delta": { + "role": "assistant", + "reasoning": "Let me think...", + }, + "index": 0, + } + ], + } + result = handler.chunk_parser(chunk) + assert result.choices[0]["delta"]["reasoning_content"] == "Let me think..." + + def test_streaming_legacy_reasoning_content_unchanged(self): + """Legacy `reasoning_content` field should pass through untouched.""" + handler = OVHCloudChatCompletionStreamingHandler( + streaming_response=iter([]), + sync_stream=True, + ) + chunk = { + "id": "test-id", + "created": 1234567890, + "model": "test-model", + "choices": [ + { + "delta": { + "role": "assistant", + "reasoning_content": "Already correct field.", + }, + "index": 0, + } + ], + } + result = handler.chunk_parser(chunk) + assert result.choices[0]["delta"]["reasoning_content"] == "Already correct field." + + def test_streaming_both_fields_legacy_wins(self): + """When both fields present, existing `reasoning_content` is not overwritten.""" + handler = OVHCloudChatCompletionStreamingHandler( + streaming_response=iter([]), + sync_stream=True, + ) + chunk = { + "id": "test-id", + "created": 1234567890, + "model": "test-model", + "choices": [ + { + "delta": { + "reasoning": "new field", + "reasoning_content": "legacy field", + }, + "index": 0, + } + ], + } + result = handler.chunk_parser(chunk) + assert result.choices[0]["delta"]["reasoning_content"] == "legacy field" \ No newline at end of file From e55e73d69b0be420a7092fcfa1159db2b7bd2d0e Mon Sep 17 00:00:00 2001 From: KunalG67 Date: Mon, 27 Apr 2026 17:37:27 +0530 Subject: [PATCH 02/85] fix(ovhcloud): use explicit None check for seconds field in STT response Replaces falsy or with explicit is not None check so that a valid seconds=0.0 value is not silently dropped during field migration. Addresses Greptile review feedback on #26595 --- .../audio_transcription/transformation.py | 6 +++++- ...hcloud_audio_transcription_transformation.py | 17 ++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/litellm/llms/ovhcloud/audio_transcription/transformation.py b/litellm/llms/ovhcloud/audio_transcription/transformation.py index e3c8308d507..f49f31d7ecd 100644 --- a/litellm/llms/ovhcloud/audio_transcription/transformation.py +++ b/litellm/llms/ovhcloud/audio_transcription/transformation.py @@ -160,7 +160,11 @@ class OVHCloudAudioTranscriptionConfig(BaseAudioTranscriptionConfig): # `duration` is replaced by `seconds` in STT responses. # Prefer `seconds`, fall back to `duration`, normalize to `duration` # so downstream consumers see a consistent key. - duration = response_json.get("seconds") or response_json.get("duration") + duration = ( + response_json["seconds"] + if "seconds" in response_json and response_json["seconds"] is not None + else response_json.get("duration") + ) if duration is not None: response_json["duration"] = duration diff --git a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py index e9abf50ba75..c8751fb2d95 100644 --- a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py +++ b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py @@ -96,4 +96,19 @@ class TestOVHCloudDurationFieldMigration: result = config.transform_audio_transcription_response(mock_response) assert result.text == "Hello world" - assert result._hidden_params["duration"] == 2.71 \ No newline at end of file + assert result._hidden_params["duration"] == 2.71 + + + + def test_seconds_zero_mapped_to_duration(self): + """seconds=0.0 must not be treated as falsy and lost.""" + from litellm.llms.ovhcloud.audio_transcription.transformation import ( + OVHCloudAudioTranscriptionConfig, + ) + from unittest.mock import MagicMock + + config = OVHCloudAudioTranscriptionConfig() + mock_response = MagicMock() + mock_response.json.return_value = {"text": "silence", "seconds": 0.0} + result = config.transform_audio_transcription_response(mock_response) + assert result._hidden_params["duration"] == 0.0 \ No newline at end of file From c0da139540345e1319c5936435f611a6aa307c44 Mon Sep 17 00:00:00 2001 From: Aneesh-Fiddler Date: Mon, 27 Apr 2026 22:51:39 +0530 Subject: [PATCH 03/85] fix(otel): populate gen_ai.output.messages and gen_ai.system_instructions for Responses API Fixes #25840 The OTel integration's set_attributes() method never populates gen_ai.output.messages, gen_ai.system_instructions, or gen_ai.response.finish_reasons for /v1/responses calls because ResponsesAPIResponse uses 'output' instead of 'choices' and the system prompt arrives as 'instructions' instead of 'system_instructions'. Changes: - Add elif branch for response_obj.get('output') to extract response text from Responses API output items (type='message'/output_text) and tool calls (type='function_call') - Coalesce system_instructions/instructions/system kwargs so the system prompt is captured for Responses API, Anthropic Messages API, and Vertex AI Gemini paths - Handle plain-string system prompts without unnecessary wrapping - Extract response_obj.get('status') as finish reason for Responses API - Add _transform_responses_api_output_to_otel() method --- litellm/integrations/opentelemetry.py | 111 +++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 10 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index b6d91d0b76d..b26850657d3 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -1678,17 +1678,35 @@ class OpenTelemetry(CustomLogger): value=safe_dumps(transformed_messages), ) - if kwargs.get("system_instructions"): - transformed_system_instructions = ( - self._transform_messages_to_otel_semantic_conventions( - kwargs.get("system_instructions") + # Coalesce the different kwarg names that carry the system + # prompt depending on the call path: + # - "system_instructions" — Vertex AI Gemini chat-completion + # - "instructions" — OpenAI Responses API + # - "system" — Anthropic Messages API + system_instructions = ( + kwargs.get("system_instructions") + or kwargs.get("instructions") + or kwargs.get("system") + ) + if system_instructions: + if isinstance(system_instructions, str): + # Plain text system prompt — no transformation needed + self.safe_set_attribute( + span=span, + key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value, + value=system_instructions, + ) + else: + transformed_system_instructions = ( + self._transform_messages_to_otel_semantic_conventions( + system_instructions + ) + ) + self.safe_set_attribute( + span=span, + key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value, + value=safe_dumps(transformed_system_instructions), ) - ) - self.safe_set_attribute( - span=span, - key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value, - value=safe_dumps(transformed_system_instructions), - ) self.safe_set_attribute( span=span, @@ -1747,6 +1765,32 @@ class OpenTelemetry(CustomLogger): value=value, ) + elif response_obj.get("output"): + # Responses API: ResponsesAPIResponse has an "output" + # list instead of "choices". Each item with + # type="message" contains a "content" list of + # OutputText objects (type="output_text"). + output_messages = ( + self._transform_responses_api_output_to_otel( + response_obj.get("output") + ) + ) + if output_messages: + self.safe_set_attribute( + span=span, + key=SpanAttributes.GEN_AI_OUTPUT_MESSAGES.value, + value=safe_dumps(output_messages), + ) + + # Extract finish reason from ResponsesAPIResponse.status + status = response_obj.get("status") + if status: + self.safe_set_attribute( + span=span, + key=SpanAttributes.GEN_AI_RESPONSE_FINISH_REASONS.value, + value=safe_dumps([status]), + ) + except Exception as e: self.handle_callback_failure( callback_name=self.callback_name or "opentelemetry" @@ -1842,6 +1886,53 @@ class OpenTelemetry(CustomLogger): transformed.append(transformed_msg) return transformed + def _transform_responses_api_output_to_otel( + self, output: List[dict] + ) -> List[dict]: + """ + Transform Responses API output items into OTEL GenAI 1.38 format. + + The Responses API returns output as a list of items, each with a + ``type`` field. Message items (``type="message"``) contain a + ``content`` list of ``OutputText`` objects with ``type="output_text"`` + and ``text`` fields. + + This method converts them to the same ``{"role": ..., "parts": [...]}`` + format used by ``_transform_choices_to_otel_semantic_conventions``. + """ + transformed = [] + for item in output: + if not isinstance(item, dict): + continue + if item.get("type") == "message": + role = item.get("role", "assistant") + parts = [] + for content in item.get("content", []): + if not isinstance(content, dict): + continue + if content.get("type") == "output_text": + text = content.get("text", "") + if text: + parts.append({"type": "text", "content": text}) + if parts: + transformed.append({"role": role, "parts": parts}) + elif item.get("type") == "function_call": + # Surface tool calls from Responses API output + tool_call = { + "role": "assistant", + "parts": [ + { + "type": "tool_call", + "name": item.get("name", ""), + "arguments": item.get("arguments", ""), + } + ], + } + if item.get("call_id"): + tool_call["parts"][0]["id"] = item["call_id"] + transformed.append(tool_call) + return transformed + def set_raw_request_attributes(self, span: Span, kwargs, response_obj): try: # Only set provider-specific raw payload attributes on this span. From 4d2e13c9070b74b0945613516e09917e15421023 Mon Sep 17 00:00:00 2001 From: Aneesh-Fiddler Date: Mon, 27 Apr 2026 23:12:34 +0530 Subject: [PATCH 04/85] test(otel): add tests for Responses API output messages, system instructions, and finish reasons Add 21 tests covering the new Responses API OTel attribute handling: TestOpenTelemetryResponsesAPI (13 tests): - gen_ai.output.messages from output items (text, function_call, mixed, multi-part) - gen_ai.response.finish_reasons from ResponsesAPIResponse.status - gen_ai.system_instructions from instructions/system/system_instructions kwargs - Precedence and absence edge cases - Regression test for existing choices-based responses TestTransformResponsesAPIOutput (8 tests): - Message with output_text, function_call items, unknown types - Edge cases: empty output, empty text, missing call_id, default role, non-dict items --- .../integrations/test_opentelemetry.py | 456 ++++++++++++++++++ 1 file changed, 456 insertions(+) diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index f7106471894..5f294005641 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -2859,3 +2859,459 @@ class TestResponseIdFallback(unittest.TestCase): otel.set_attributes(mock_span, kwargs, response_obj) mock_span.set_attribute.assert_any_call("litellm.call_id", call_id) + + + +class TestOpenTelemetryResponsesAPI(unittest.TestCase): + """ + Tests for Responses API (/v1/responses) OTel span attributes. + + The Responses API uses ``output`` (list of output items) instead of + ``choices``, ``instructions`` instead of ``system_instructions``, and + ``status`` instead of per-choice ``finish_reason``. + + See: https://github.com/BerriAI/litellm/issues/25840 + """ + + def _base_kwargs(self, **overrides): + """Return minimal kwargs for set_attributes with Responses API defaults.""" + kwargs = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "What is 2+2?"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "resp_abc123", + "call_type": "responses", + "metadata": {}, + }, + } + kwargs.update(overrides) + return kwargs + + def _responses_api_response_obj(self, text="The answer is 4.", status="completed"): + """Return a dict mimicking ResponsesAPIResponse with a message output.""" + return { + "id": "resp_abc123", + "model": "gpt-4o", + "status": status, + "output": [ + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": text, + } + ], + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30, + }, + } + + def _get_attr(self, mock_span, attr_name): + """Extract the value set for a specific attribute name, or None.""" + calls = [ + call + for call in mock_span.set_attribute.call_args_list + if call[0][0] == attr_name + ] + if not calls: + return None + return calls[0][0][1] + + # ------------------------------------------------------------------ + # gen_ai.output.messages + # ------------------------------------------------------------------ + + def test_output_messages_populated_for_responses_api(self): + """gen_ai.output.messages must be set when response has output items.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = self._base_kwargs() + response_obj = self._responses_api_response_obj(text="The answer is 4.") + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + raw = self._get_attr(mock_span, "gen_ai.output.messages") + self.assertIsNotNone(raw, "gen_ai.output.messages should be set") + + parsed = json.loads(raw) + self.assertIsInstance(parsed, list) + self.assertEqual(len(parsed), 1) + self.assertEqual(parsed[0]["role"], "assistant") + self.assertIn("parts", parsed[0]) + self.assertEqual(parsed[0]["parts"][0]["type"], "text") + self.assertEqual(parsed[0]["parts"][0]["content"], "The answer is 4.") + + def test_output_messages_with_multiple_content_items(self): + """Multiple output_text items in a single message should all appear as parts.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + response_obj = { + "id": "resp_multi", + "model": "gpt-4o", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [ + {"type": "output_text", "text": "First paragraph."}, + {"type": "output_text", "text": "Second paragraph."}, + ], + } + ], + } + + otel.set_attributes( + span=mock_span, kwargs=self._base_kwargs(), response_obj=response_obj + ) + + raw = self._get_attr(mock_span, "gen_ai.output.messages") + parsed = json.loads(raw) + self.assertEqual(len(parsed[0]["parts"]), 2) + self.assertEqual(parsed[0]["parts"][0]["content"], "First paragraph.") + self.assertEqual(parsed[0]["parts"][1]["content"], "Second paragraph.") + + def test_output_messages_with_function_call(self): + """function_call output items should appear as tool_call parts.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + response_obj = { + "id": "resp_fc", + "model": "gpt-4o", + "status": "completed", + "output": [ + { + "type": "function_call", + "name": "get_weather", + "call_id": "call_abc", + "arguments": '{"location": "SF"}', + } + ], + } + + otel.set_attributes( + span=mock_span, kwargs=self._base_kwargs(), response_obj=response_obj + ) + + raw = self._get_attr(mock_span, "gen_ai.output.messages") + parsed = json.loads(raw) + self.assertEqual(len(parsed), 1) + self.assertEqual(parsed[0]["role"], "assistant") + self.assertEqual(parsed[0]["parts"][0]["type"], "tool_call") + self.assertEqual(parsed[0]["parts"][0]["name"], "get_weather") + self.assertEqual(parsed[0]["parts"][0]["arguments"], '{"location": "SF"}') + self.assertEqual(parsed[0]["parts"][0]["id"], "call_abc") + + def test_output_messages_mixed_message_and_function_call(self): + """Mixed output with both message and function_call items.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + response_obj = { + "id": "resp_mixed", + "model": "gpt-4o", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [ + {"type": "output_text", "text": "Let me check the weather."}, + ], + }, + { + "type": "function_call", + "name": "get_weather", + "call_id": "call_xyz", + "arguments": "{}", + }, + ], + } + + otel.set_attributes( + span=mock_span, kwargs=self._base_kwargs(), response_obj=response_obj + ) + + raw = self._get_attr(mock_span, "gen_ai.output.messages") + parsed = json.loads(raw) + self.assertEqual(len(parsed), 2) + self.assertEqual(parsed[0]["role"], "assistant") + self.assertEqual(parsed[0]["parts"][0]["content"], "Let me check the weather.") + self.assertEqual(parsed[1]["parts"][0]["type"], "tool_call") + + def test_output_messages_empty_text_skipped(self): + """Output items with empty text should not produce parts.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + response_obj = { + "id": "resp_empty", + "model": "gpt-4o", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": ""}], + } + ], + } + + otel.set_attributes( + span=mock_span, kwargs=self._base_kwargs(), response_obj=response_obj + ) + + # No output messages should be set since the text is empty + raw = self._get_attr(mock_span, "gen_ai.output.messages") + self.assertIsNone(raw, "Empty output text should not produce gen_ai.output.messages") + + def test_choices_still_work(self): + """Existing choices-based responses must still work (no regression).""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + }, + } + + response_obj = { + "id": "chatcmpl-123", + "model": "gpt-4", + "choices": [ + { + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Hi there!"}, + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 10, "total_tokens": 15}, + } + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + raw = self._get_attr(mock_span, "gen_ai.output.messages") + parsed = json.loads(raw) + self.assertEqual(parsed[0]["parts"][0]["content"], "Hi there!") + self.assertEqual(parsed[0]["finish_reason"], "stop") + + # ------------------------------------------------------------------ + # gen_ai.response.finish_reasons + # ------------------------------------------------------------------ + + def test_finish_reasons_from_status(self): + """gen_ai.response.finish_reasons should use ResponsesAPIResponse.status.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + otel.set_attributes( + span=mock_span, + kwargs=self._base_kwargs(), + response_obj=self._responses_api_response_obj(status="completed"), + ) + + raw = self._get_attr(mock_span, "gen_ai.response.finish_reasons") + self.assertIsNotNone(raw) + parsed = json.loads(raw) + self.assertEqual(parsed, ["completed"]) + + def test_finish_reasons_incomplete_status(self): + """Non-completed status values should still be captured.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + otel.set_attributes( + span=mock_span, + kwargs=self._base_kwargs(), + response_obj=self._responses_api_response_obj(status="incomplete"), + ) + + raw = self._get_attr(mock_span, "gen_ai.response.finish_reasons") + parsed = json.loads(raw) + self.assertEqual(parsed, ["incomplete"]) + + # ------------------------------------------------------------------ + # gen_ai.system_instructions + # ------------------------------------------------------------------ + + def test_system_instructions_from_instructions_kwarg(self): + """Responses API passes system prompt as kwargs['instructions'].""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = self._base_kwargs(instructions="You are a math tutor.") + response_obj = self._responses_api_response_obj() + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + value = self._get_attr(mock_span, "gen_ai.system_instructions") + self.assertEqual(value, "You are a math tutor.") + + def test_system_instructions_from_system_kwarg(self): + """Anthropic Messages API passes system prompt as kwargs['system'].""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = self._base_kwargs(system="You are a helpful assistant.") + response_obj = self._responses_api_response_obj() + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + value = self._get_attr(mock_span, "gen_ai.system_instructions") + self.assertEqual(value, "You are a helpful assistant.") + + def test_system_instructions_from_system_instructions_kwarg(self): + """Vertex AI Gemini path uses kwargs['system_instructions'] (existing behavior).""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = self._base_kwargs( + system_instructions=[{"role": "system", "content": "Be concise."}] + ) + response_obj = self._responses_api_response_obj() + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + raw = self._get_attr(mock_span, "gen_ai.system_instructions") + self.assertIsNotNone(raw) + parsed = json.loads(raw) + self.assertEqual(parsed[0]["role"], "system") + self.assertIn("parts", parsed[0]) + + def test_system_instructions_precedence(self): + """system_instructions takes precedence over instructions and system.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = self._base_kwargs( + system_instructions="From Gemini", + instructions="From Responses API", + system="From Anthropic", + ) + response_obj = self._responses_api_response_obj() + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + # system_instructions (string) should win — it's checked first + value = self._get_attr(mock_span, "gen_ai.system_instructions") + self.assertEqual(value, "From Gemini") + + def test_no_system_instructions_when_absent(self): + """No gen_ai.system_instructions attr when none of the kwargs are set.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = self._base_kwargs() + response_obj = self._responses_api_response_obj() + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + value = self._get_attr(mock_span, "gen_ai.system_instructions") + self.assertIsNone(value) + + +class TestTransformResponsesAPIOutput(unittest.TestCase): + """ + Unit tests for _transform_responses_api_output_to_otel. + """ + + def test_message_with_output_text(self): + otel = OpenTelemetry() + output = [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello!"}], + } + ] + result = otel._transform_responses_api_output_to_otel(output) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["role"], "assistant") + self.assertEqual(result[0]["parts"], [{"type": "text", "content": "Hello!"}]) + + def test_function_call_item(self): + otel = OpenTelemetry() + output = [ + { + "type": "function_call", + "name": "search", + "call_id": "call_1", + "arguments": '{"q": "test"}', + } + ] + result = otel._transform_responses_api_output_to_otel(output) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["role"], "assistant") + self.assertEqual(result[0]["parts"][0]["type"], "tool_call") + self.assertEqual(result[0]["parts"][0]["name"], "search") + self.assertEqual(result[0]["parts"][0]["id"], "call_1") + + def test_function_call_without_call_id(self): + otel = OpenTelemetry() + output = [ + { + "type": "function_call", + "name": "search", + "arguments": "{}", + } + ] + result = otel._transform_responses_api_output_to_otel(output) + self.assertNotIn("id", result[0]["parts"][0]) + + def test_unknown_type_ignored(self): + otel = OpenTelemetry() + output = [{"type": "reasoning", "content": "thinking..."}] + result = otel._transform_responses_api_output_to_otel(output) + self.assertEqual(result, []) + + def test_non_dict_items_ignored(self): + otel = OpenTelemetry() + output = ["not a dict", 42, None] + result = otel._transform_responses_api_output_to_otel(output) + self.assertEqual(result, []) + + def test_empty_output(self): + otel = OpenTelemetry() + result = otel._transform_responses_api_output_to_otel([]) + self.assertEqual(result, []) + + def test_message_with_empty_text_skipped(self): + otel = OpenTelemetry() + output = [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": ""}], + } + ] + result = otel._transform_responses_api_output_to_otel(output) + self.assertEqual(result, []) + + def test_message_default_role(self): + """Messages without explicit role should default to assistant.""" + otel = OpenTelemetry() + output = [ + { + "type": "message", + "content": [{"type": "output_text", "text": "Hi"}], + } + ] + result = otel._transform_responses_api_output_to_otel(output) + self.assertEqual(result[0]["role"], "assistant") From e70b0c97a4c8d6aef4efd7d1738172acfe4510ea Mon Sep 17 00:00:00 2001 From: Aneesh-Fiddler Date: Tue, 28 Apr 2026 11:26:14 +0530 Subject: [PATCH 05/85] style: apply black formatting to opentelemetry.py --- litellm/integrations/opentelemetry.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index b26850657d3..664aeadc7c2 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -1770,10 +1770,8 @@ class OpenTelemetry(CustomLogger): # list instead of "choices". Each item with # type="message" contains a "content" list of # OutputText objects (type="output_text"). - output_messages = ( - self._transform_responses_api_output_to_otel( - response_obj.get("output") - ) + output_messages = self._transform_responses_api_output_to_otel( + response_obj.get("output") ) if output_messages: self.safe_set_attribute( @@ -1886,9 +1884,7 @@ class OpenTelemetry(CustomLogger): transformed.append(transformed_msg) return transformed - def _transform_responses_api_output_to_otel( - self, output: List[dict] - ) -> List[dict]: + def _transform_responses_api_output_to_otel(self, output: List[dict]) -> List[dict]: """ Transform Responses API output items into OTEL GenAI 1.38 format. From c30d58f7e30a0568c265814d70cea2301bea51a4 Mon Sep 17 00:00:00 2001 From: Aneesh-Fiddler Date: Tue, 28 Apr 2026 12:30:39 +0530 Subject: [PATCH 06/85] fix: resolve mypy indexed assignment error in function_call handling Build the tool_call part dict separately with an explicit type annotation so mypy can track the type, avoiding the 'Unsupported target for indexed assignment' error on tool_call["parts"][0]["id"]. --- litellm/integrations/opentelemetry.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 664aeadc7c2..8084a9d0f22 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -1914,19 +1914,14 @@ class OpenTelemetry(CustomLogger): transformed.append({"role": role, "parts": parts}) elif item.get("type") == "function_call": # Surface tool calls from Responses API output - tool_call = { - "role": "assistant", - "parts": [ - { - "type": "tool_call", - "name": item.get("name", ""), - "arguments": item.get("arguments", ""), - } - ], + part: dict = { + "type": "tool_call", + "name": item.get("name", ""), + "arguments": item.get("arguments", ""), } if item.get("call_id"): - tool_call["parts"][0]["id"] = item["call_id"] - transformed.append(tool_call) + part["id"] = item["call_id"] + transformed.append({"role": "assistant", "parts": [part]}) return transformed def set_raw_request_attributes(self, span: Span, kwargs, response_obj): From 466b4ddae31beb94635f72adc796d731b485930f Mon Sep 17 00:00:00 2001 From: Aneesh-Fiddler Date: Tue, 28 Apr 2026 13:33:33 +0530 Subject: [PATCH 07/85] =?UTF-8?q?fix:=20address=20review=20comments=20?= =?UTF-8?q?=E2=80=94=20Pydantic=20compat,=20falsy=20fallthrough,=20per-too?= =?UTF-8?q?l-call=20attrs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace isinstance(item, dict) with hasattr(item, 'get') so Pydantic model instances (ResponseOutputMessage, ResponseFunctionToolCall) are accepted alongside plain dicts (P1) - Use 'is not None' guards instead of or-chain for system_instructions coalescing to prevent falsy values (e.g. []) falling through to the wrong kwarg (P2) - Emit per-tool-call span attributes (gen_ai.completion.N.function_call.*) for Responses API function_call items, matching the choices branch parity with _tool_calls_kv_pair (P2) - Add 4 new tests: Pydantic-like objects, falsy fallthrough guard, per-tool-call attribute emission, multiple tool call indexing --- litellm/integrations/opentelemetry.py | 62 ++++++- .../integrations/test_opentelemetry.py | 171 ++++++++++++++++++ 2 files changed, 227 insertions(+), 6 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 8084a9d0f22..90e647d86bf 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -1683,10 +1683,16 @@ class OpenTelemetry(CustomLogger): # - "system_instructions" — Vertex AI Gemini chat-completion # - "instructions" — OpenAI Responses API # - "system" — Anthropic Messages API + # Use `is not None` rather than truthiness to avoid falsy + # values (e.g. []) falling through to the wrong kwarg. system_instructions = ( kwargs.get("system_instructions") - or kwargs.get("instructions") - or kwargs.get("system") + if kwargs.get("system_instructions") is not None + else ( + kwargs.get("instructions") + if kwargs.get("instructions") is not None + else kwargs.get("system") + ) ) if system_instructions: if isinstance(system_instructions, str): @@ -1770,8 +1776,9 @@ class OpenTelemetry(CustomLogger): # list instead of "choices". Each item with # type="message" contains a "content" list of # OutputText objects (type="output_text"). + output_items = response_obj.get("output") output_messages = self._transform_responses_api_output_to_otel( - response_obj.get("output") + output_items ) if output_messages: self.safe_set_attribute( @@ -1780,6 +1787,43 @@ class OpenTelemetry(CustomLogger): value=safe_dumps(output_messages), ) + # Emit per-tool-call span attributes (parity with + # the choices branch that calls _tool_calls_kv_pair). + # Convert Responses API function_call items to the + # ChatCompletionMessageToolCall format expected by + # _tool_calls_kv_pair. + tool_calls = [] + for out_item in output_items: + if ( + hasattr(out_item, "get") + and out_item.get("type") == "function_call" + ): + tool_calls.append( + { + "function": { + "name": out_item.get("name", ""), + "arguments": out_item.get("arguments", ""), + } + } + ) + if tool_calls: + kv_pairs = OpenTelemetry._tool_calls_kv_pair(tool_calls) # type: ignore + for key, value in kv_pairs.items(): + self.safe_set_attribute( + span=span, + key=key, + value=value, + ) + + # Extract finish reason from ResponsesAPIResponse.status + status = response_obj.get("status") + if status: + self.safe_set_attribute( + span=span, + key=SpanAttributes.GEN_AI_RESPONSE_FINISH_REASONS.value, + value=safe_dumps([status]), + ) + # Extract finish reason from ResponsesAPIResponse.status status = response_obj.get("status") if status: @@ -1884,7 +1928,7 @@ class OpenTelemetry(CustomLogger): transformed.append(transformed_msg) return transformed - def _transform_responses_api_output_to_otel(self, output: List[dict]) -> List[dict]: + def _transform_responses_api_output_to_otel(self, output: List) -> List[dict]: """ Transform Responses API output items into OTEL GenAI 1.38 format. @@ -1893,18 +1937,24 @@ class OpenTelemetry(CustomLogger): ``content`` list of ``OutputText`` objects with ``type="output_text"`` and ``text`` fields. + Items may be plain dicts or Pydantic model instances (e.g. + ``ResponseOutputMessage``, ``ResponseFunctionToolCall``). Both + expose a ``.get()`` method via ``BaseLiteLLMOpenAIResponseObject``, + so we use ``hasattr(item, "get")`` rather than ``isinstance(item, + dict)`` to accept either form. + This method converts them to the same ``{"role": ..., "parts": [...]}`` format used by ``_transform_choices_to_otel_semantic_conventions``. """ transformed = [] for item in output: - if not isinstance(item, dict): + if not hasattr(item, "get"): continue if item.get("type") == "message": role = item.get("role", "assistant") parts = [] for content in item.get("content", []): - if not isinstance(content, dict): + if not hasattr(content, "get"): continue if content.get("type") == "output_text": text = content.get("text", "") diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 5f294005641..56aba4bc5ed 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -3315,3 +3315,174 @@ class TestTransformResponsesAPIOutput(unittest.TestCase): ] result = otel._transform_responses_api_output_to_otel(output) self.assertEqual(result[0]["role"], "assistant") + + + def test_pydantic_like_objects_accepted(self): + """Items with .get() but not isinstance(dict) should be accepted.""" + + class FakeOutputItem: + """Mimics BaseLiteLLMOpenAIResponseObject duck-typing.""" + + def __init__(self, data): + self._data = data + + def get(self, key, default=None): + return self._data.get(key, default) + + class FakeContent: + def __init__(self, data): + self._data = data + + def get(self, key, default=None): + return self._data.get(key, default) + + otel = OpenTelemetry() + output = [ + FakeOutputItem( + { + "type": "message", + "role": "assistant", + "content": [ + FakeContent({"type": "output_text", "text": "Pydantic works!"}), + ], + } + ) + ] + result = otel._transform_responses_api_output_to_otel(output) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["parts"][0]["content"], "Pydantic works!") + + +class TestSystemInstructionsPrecedence(unittest.TestCase): + """Tests for the is-not-None precedence in system_instructions coalescing.""" + + def _get_attr(self, mock_span, attr_name): + calls = [ + call + for call in mock_span.set_attribute.call_args_list + if call[0][0] == attr_name + ] + if not calls: + return None + return calls[0][0][1] + + def _base_kwargs(self, **overrides): + kwargs = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Hi"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "responses", + "metadata": {}, + }, + } + kwargs.update(overrides) + return kwargs + + def test_empty_list_system_instructions_does_not_fallthrough(self): + """An empty list for system_instructions should NOT fall through to instructions.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = self._base_kwargs( + system_instructions=[], + instructions="Should not be used", + ) + response_obj = {"id": "r1", "model": "gpt-4o"} + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + # system_instructions is [] (falsy but not None), so it wins. + # Since it's an empty list, no attribute should be set (nothing to transform). + value = self._get_attr(mock_span, "gen_ai.system_instructions") + # The empty list is truthy for `is not None` but produces empty + # transformed output — the attribute should NOT contain "Should not be used". + if value is not None: + self.assertNotIn("Should not be used", str(value)) + + +class TestResponsesAPIToolCallSpanAttributes(unittest.TestCase): + """Tests for per-tool-call span attributes on Responses API function_call items.""" + + def _base_kwargs(self): + return { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "What is the weather?"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "resp_tc", + "call_type": "responses", + "metadata": {}, + }, + } + + def test_per_tool_call_attributes_emitted(self): + """function_call output items should produce per-tool-call span attributes.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + response_obj = { + "id": "resp_tc", + "model": "gpt-4o", + "status": "completed", + "output": [ + { + "type": "function_call", + "name": "get_weather", + "call_id": "call_abc", + "arguments": '{"location": "SF"}', + } + ], + } + + otel.set_attributes(span=mock_span, kwargs=self._base_kwargs(), response_obj=response_obj) + + # Verify per-tool-call attributes were set (same format as choices branch) + attr_names = [call[0][0] for call in mock_span.set_attribute.call_args_list] + tool_call_attrs = [a for a in attr_names if "function_call" in a] + self.assertTrue(len(tool_call_attrs) > 0, "Per-tool-call span attributes should be emitted") + + # Verify the name attribute specifically + mock_span.set_attribute.assert_any_call( + "gen_ai.completion.0.function_call.name", "get_weather" + ) + mock_span.set_attribute.assert_any_call( + "gen_ai.completion.0.function_call.arguments", '{"location": "SF"}' + ) + + def test_multiple_tool_calls_indexed(self): + """Multiple function_call items should be indexed correctly.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + response_obj = { + "id": "resp_tc2", + "model": "gpt-4o", + "status": "completed", + "output": [ + { + "type": "function_call", + "name": "get_weather", + "call_id": "call_1", + "arguments": "{}", + }, + { + "type": "function_call", + "name": "get_time", + "call_id": "call_2", + "arguments": "{}", + }, + ], + } + + otel.set_attributes(span=mock_span, kwargs=self._base_kwargs(), response_obj=response_obj) + + mock_span.set_attribute.assert_any_call( + "gen_ai.completion.0.function_call.name", "get_weather" + ) + mock_span.set_attribute.assert_any_call( + "gen_ai.completion.1.function_call.name", "get_time" + ) From 982fed46321041c4bc46d02312aa731d6662bd88 Mon Sep 17 00:00:00 2001 From: KunalG67 Date: Tue, 28 Apr 2026 18:09:12 +0530 Subject: [PATCH 08/85] fix(ovhcloud): handle reasoning field migration in non-streaming responses Adds transform_response to OVHCloudChatConfig to normalise the new easoning field to easoning_content in non-streaming responses, matching the existing streaming fix in chunk_parser. Addresses maintainer feedback on #26595 --- litellm/llms/ovhcloud/chat/transformation.py | 50 ++++++++++++++++-- .../test_ovhcloud_chat_transformation.py | 52 ++++++++++++++++++- 2 files changed, 97 insertions(+), 5 deletions(-) diff --git a/litellm/llms/ovhcloud/chat/transformation.py b/litellm/llms/ovhcloud/chat/transformation.py index 4100c548f2c..77d3683566c 100644 --- a/litellm/llms/ovhcloud/chat/transformation.py +++ b/litellm/llms/ovhcloud/chat/transformation.py @@ -5,17 +5,17 @@ Our unified API follows the OpenAI standard. More information on our website: https://endpoints.ai.cloud.ovh.net """ -from typing import Optional, Union, List +from typing import Any, Optional, Union, List import httpx -from litellm.utils import ModelResponseStream +from litellm.utils import ModelResponse, ModelResponseStream from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.ovhcloud.utils import OVHCloudException from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj from litellm.types.llms.openai import AllMessageValues - class OVHCloudChatConfig(OpenAIGPTConfig): @property def custom_llm_provider(self) -> Optional[str]: @@ -75,6 +75,50 @@ class OVHCloudChatConfig(OpenAIGPTConfig): return response + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + # Call parent to do standard OpenAI response parsing + model_response = super().transform_response( + model=model, + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data=request_data, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + encoding=encoding, + api_key=api_key, + json_mode=json_mode, + ) + + # OVHCloud field migration (deadline: 2026-05-11): + # `reasoning_content` is replaced by `reasoning` in non-streaming responses. + # Normalise to `reasoning_content` so downstream consumers + # see a consistent key during the transition window. + for choice in model_response.choices: + message = getattr(choice, "message", None) + if message is not None: + reasoning_new = getattr(message, "reasoning", None) + reasoning_legacy = getattr(message, "reasoning_content", None) + if reasoning_new is not None and reasoning_legacy is None: + message.reasoning_content = reasoning_new + + return model_response + + class OVHCloudChatCompletionStreamingHandler(BaseModelResponseIterator): """ Handler for OVHCloud AI Endpoints streaming chat completion responses diff --git a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py index 88ce3b4c296..b112f6d87f1 100644 --- a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py +++ b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py @@ -4,7 +4,7 @@ Unit tests for OVHCloud AI Endpoints chat integration. import os import sys - +import litellm import pytest from litellm.llms.ovhcloud.utils import OVHCloudException @@ -364,4 +364,52 @@ class TestOVHCloudReasoningFieldMigration: ], } result = handler.chunk_parser(chunk) - assert result.choices[0]["delta"]["reasoning_content"] == "legacy field" \ No newline at end of file + assert result.choices[0]["delta"]["reasoning_content"] == "legacy field" + + + def test_non_streaming_new_reasoning_field(self): + """Non-streaming: new `reasoning` field should be mapped to `reasoning_content`.""" + from unittest.mock import MagicMock, patch + import json + + config = OVHCloudChatConfig() + + raw_response = MagicMock() + raw_response.status_code = 200 + raw_response.headers = {"Content-Type": "application/json"} + raw_response.text = json.dumps({ + "id": "test-id", + "object": "chat.completion", + "created": 1234567890, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello!", + "reasoning": "Let me think...", + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }) + raw_response.json.return_value = json.loads(raw_response.text) + + model_response = litellm.ModelResponse() + + result = config.transform_response( + model="ovhcloud/test-model", + raw_response=raw_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + api_key="test-key", + ) + + assert result.choices[0].message.reasoning_content == "Let me think..." \ No newline at end of file From 8f48d880da974e349deb63a47363a12c618a5301 Mon Sep 17 00:00:00 2001 From: KunalG67 Date: Tue, 28 Apr 2026 18:25:11 +0530 Subject: [PATCH 09/85] style: apply black formatting to ovhcloud chat transformation --- litellm/llms/ovhcloud/chat/transformation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/ovhcloud/chat/transformation.py b/litellm/llms/ovhcloud/chat/transformation.py index 77d3683566c..b5752d16309 100644 --- a/litellm/llms/ovhcloud/chat/transformation.py +++ b/litellm/llms/ovhcloud/chat/transformation.py @@ -16,6 +16,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj from litellm.types.llms.openai import AllMessageValues + class OVHCloudChatConfig(OpenAIGPTConfig): @property def custom_llm_provider(self) -> Optional[str]: @@ -74,7 +75,6 @@ class OVHCloudChatConfig(OpenAIGPTConfig): response.update(extra_body) return response - def transform_response( self, model: str, @@ -116,7 +116,7 @@ class OVHCloudChatConfig(OpenAIGPTConfig): if reasoning_new is not None and reasoning_legacy is None: message.reasoning_content = reasoning_new - return model_response + return model_response class OVHCloudChatCompletionStreamingHandler(BaseModelResponseIterator): From 9928618788389e623aa0c57b9249d084bfe72482 Mon Sep 17 00:00:00 2001 From: Aneesh-Fiddler Date: Tue, 28 Apr 2026 19:58:58 +0530 Subject: [PATCH 10/85] fix: remove duplicate gen_ai.response.finish_reasons block --- litellm/integrations/opentelemetry.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 90e647d86bf..d116fc44658 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -1824,15 +1824,6 @@ class OpenTelemetry(CustomLogger): value=safe_dumps([status]), ) - # Extract finish reason from ResponsesAPIResponse.status - status = response_obj.get("status") - if status: - self.safe_set_attribute( - span=span, - key=SpanAttributes.GEN_AI_RESPONSE_FINISH_REASONS.value, - value=safe_dumps([status]), - ) - except Exception as e: self.handle_callback_failure( callback_name=self.callback_name or "opentelemetry" From 90bcd232c37389397cdcb763737f150899f1f722 Mon Sep 17 00:00:00 2001 From: KunalG67 Date: Tue, 28 Apr 2026 23:09:17 +0530 Subject: [PATCH 11/85] fix(ovhcloud): remove dead transform_response override The parent OpenAIGPTConfig already handles reasoning->reasoning_content for non-streaming via _extract_reasoning_content. The override was dead code giving false confidence. Streaming fix in chunk_parser is the only change needed for chat completions. Addresses Agent Shin review feedback on #26595 --- litellm/llms/ovhcloud/chat/transformation.py | 47 ++---------------- .../test_ovhcloud_chat_transformation.py | 48 +------------------ 2 files changed, 4 insertions(+), 91 deletions(-) diff --git a/litellm/llms/ovhcloud/chat/transformation.py b/litellm/llms/ovhcloud/chat/transformation.py index b5752d16309..140cb855323 100644 --- a/litellm/llms/ovhcloud/chat/transformation.py +++ b/litellm/llms/ovhcloud/chat/transformation.py @@ -5,15 +5,15 @@ Our unified API follows the OpenAI standard. More information on our website: https://endpoints.ai.cloud.ovh.net """ -from typing import Any, Optional, Union, List +from typing import Optional, Union, List import httpx -from litellm.utils import ModelResponse, ModelResponseStream +from litellm.utils import ModelResponseStream from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.ovhcloud.utils import OVHCloudException from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj + from litellm.types.llms.openai import AllMessageValues @@ -75,48 +75,7 @@ class OVHCloudChatConfig(OpenAIGPTConfig): response.update(extra_body) return response - def transform_response( - self, - model: str, - raw_response: httpx.Response, - model_response: ModelResponse, - logging_obj: LiteLLMLoggingObj, - request_data: dict, - messages: List[AllMessageValues], - optional_params: dict, - litellm_params: dict, - encoding: Any, - api_key: Optional[str] = None, - json_mode: Optional[bool] = None, - ) -> ModelResponse: - # Call parent to do standard OpenAI response parsing - model_response = super().transform_response( - model=model, - raw_response=raw_response, - model_response=model_response, - logging_obj=logging_obj, - request_data=request_data, - messages=messages, - optional_params=optional_params, - litellm_params=litellm_params, - encoding=encoding, - api_key=api_key, - json_mode=json_mode, - ) - # OVHCloud field migration (deadline: 2026-05-11): - # `reasoning_content` is replaced by `reasoning` in non-streaming responses. - # Normalise to `reasoning_content` so downstream consumers - # see a consistent key during the transition window. - for choice in model_response.choices: - message = getattr(choice, "message", None) - if message is not None: - reasoning_new = getattr(message, "reasoning", None) - reasoning_legacy = getattr(message, "reasoning_content", None) - if reasoning_new is not None and reasoning_legacy is None: - message.reasoning_content = reasoning_new - - return model_response class OVHCloudChatCompletionStreamingHandler(BaseModelResponseIterator): diff --git a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py index b112f6d87f1..40d57c76d02 100644 --- a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py +++ b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py @@ -4,7 +4,7 @@ Unit tests for OVHCloud AI Endpoints chat integration. import os import sys -import litellm + import pytest from litellm.llms.ovhcloud.utils import OVHCloudException @@ -367,49 +367,3 @@ class TestOVHCloudReasoningFieldMigration: assert result.choices[0]["delta"]["reasoning_content"] == "legacy field" - def test_non_streaming_new_reasoning_field(self): - """Non-streaming: new `reasoning` field should be mapped to `reasoning_content`.""" - from unittest.mock import MagicMock, patch - import json - - config = OVHCloudChatConfig() - - raw_response = MagicMock() - raw_response.status_code = 200 - raw_response.headers = {"Content-Type": "application/json"} - raw_response.text = json.dumps({ - "id": "test-id", - "object": "chat.completion", - "created": 1234567890, - "model": "test-model", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Hello!", - "reasoning": "Let me think...", - }, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, - }) - raw_response.json.return_value = json.loads(raw_response.text) - - model_response = litellm.ModelResponse() - - result = config.transform_response( - model="ovhcloud/test-model", - raw_response=raw_response, - model_response=model_response, - logging_obj=MagicMock(), - request_data={}, - messages=[], - optional_params={}, - litellm_params={}, - encoding=None, - api_key="test-key", - ) - - assert result.choices[0].message.reasoning_content == "Let me think..." \ No newline at end of file From d73e24c1f93664dc147ce8ef2c5a2ffcef6f9eb7 Mon Sep 17 00:00:00 2001 From: KunalG67 Date: Tue, 28 Apr 2026 23:19:13 +0530 Subject: [PATCH 12/85] fix(ovhcloud): remove dead transform_response override, parent already handles non-streaming via _extract_reasoning_content --- litellm/llms/ovhcloud/chat/transformation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/litellm/llms/ovhcloud/chat/transformation.py b/litellm/llms/ovhcloud/chat/transformation.py index 140cb855323..62f51f1e9da 100644 --- a/litellm/llms/ovhcloud/chat/transformation.py +++ b/litellm/llms/ovhcloud/chat/transformation.py @@ -76,8 +76,6 @@ class OVHCloudChatConfig(OpenAIGPTConfig): return response - - class OVHCloudChatCompletionStreamingHandler(BaseModelResponseIterator): """ Handler for OVHCloud AI Endpoints streaming chat completion responses From 53102529ca6e817cd398f23bcc0bd63501c9f169 Mon Sep 17 00:00:00 2001 From: Aneesh-Fiddler Date: Wed, 29 Apr 2026 19:00:43 +0530 Subject: [PATCH 13/85] ci: retrigger checks after retargeting to litellm_oss_staging_04_27_2026 From c319a19c25d746dbcfd27ab3c69984f5d66eb358 Mon Sep 17 00:00:00 2001 From: Aneesh-Fiddler Date: Thu, 30 Apr 2026 11:04:08 +0530 Subject: [PATCH 14/85] fix: handle raw Pydantic v2 models from openai SDK in output transformation The openai SDK returns ResponseOutputMessage and ResponseOutputText as raw Pydantic v2 models that lack .get() (unlike LiteLLM's own wrapper objects). Add a _to_dict() helper that normalizes plain dicts, BaseLiteLLMOpenAIResponseObject (has .get()), and raw Pydantic models (has .model_dump()) into a consistent dict interface. --- litellm/integrations/opentelemetry.py | 54 +++++++++++++++++++-------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index d116fc44658..a12d67de4b4 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -1794,15 +1794,13 @@ class OpenTelemetry(CustomLogger): # _tool_calls_kv_pair. tool_calls = [] for out_item in output_items: - if ( - hasattr(out_item, "get") - and out_item.get("type") == "function_call" - ): + item_d = self._to_dict(out_item) + if item_d and item_d.get("type") == "function_call": tool_calls.append( { "function": { - "name": out_item.get("name", ""), - "arguments": out_item.get("arguments", ""), + "name": item_d.get("name", ""), + "arguments": item_d.get("arguments", ""), } } ) @@ -1919,6 +1917,31 @@ class OpenTelemetry(CustomLogger): transformed.append(transformed_msg) return transformed + @staticmethod + def _to_dict(obj) -> Optional[dict]: + """Normalize an object to a plain dict. + + Handles three forms that appear in practice: + + 1. Plain ``dict`` — returned as-is. + 2. LiteLLM's ``BaseLiteLLMOpenAIResponseObject`` — exposes a + ``.get()`` method that delegates to ``__dict__``. + 3. Raw Pydantic v2 models from the ``openai`` SDK (e.g. + ``ResponseOutputMessage``, ``ResponseOutputText``) — these do + **not** have ``.get()`` but do have ``.model_dump()``. + + Returns ``None`` for anything else so callers can skip it. + """ + if isinstance(obj, dict): + return obj + if hasattr(obj, "get"): + # BaseLiteLLMOpenAIResponseObject duck-type + return obj # type: ignore[return-value] + if hasattr(obj, "model_dump"): + # Raw Pydantic v2 model (e.g. openai SDK types) + return obj.model_dump() # type: ignore[union-attr] + return None + def _transform_responses_api_output_to_otel(self, output: List) -> List[dict]: """ Transform Responses API output items into OTEL GenAI 1.38 format. @@ -1928,24 +1951,25 @@ class OpenTelemetry(CustomLogger): ``content`` list of ``OutputText`` objects with ``type="output_text"`` and ``text`` fields. - Items may be plain dicts or Pydantic model instances (e.g. - ``ResponseOutputMessage``, ``ResponseFunctionToolCall``). Both - expose a ``.get()`` method via ``BaseLiteLLMOpenAIResponseObject``, - so we use ``hasattr(item, "get")`` rather than ``isinstance(item, - dict)`` to accept either form. + Items may be plain dicts, LiteLLM wrapper objects (with ``.get()``), + or raw Pydantic v2 models from the ``openai`` SDK (with + ``.model_dump()``). We normalize each item to a dict via + ``_to_dict`` before processing. This method converts them to the same ``{"role": ..., "parts": [...]}`` format used by ``_transform_choices_to_otel_semantic_conventions``. """ transformed = [] - for item in output: - if not hasattr(item, "get"): + for raw_item in output: + item = self._to_dict(raw_item) + if item is None: continue if item.get("type") == "message": role = item.get("role", "assistant") parts = [] - for content in item.get("content", []): - if not hasattr(content, "get"): + for raw_content in item.get("content", []): + content = self._to_dict(raw_content) + if content is None: continue if content.get("type") == "output_text": text = content.get("text", "") From 209bd0b9061f7b602d148f8d6da6ffddc2014c87 Mon Sep 17 00:00:00 2001 From: pnookala-godaddy Date: Tue, 5 May 2026 12:18:44 -0700 Subject: [PATCH 15/85] fix(proxy): sort spend updates to prevent DB deadlocks Iterate user/key/team/team_member/org/end_user/tag spend dicts in sorted order inside each Prisma transaction so concurrent pods acquire row locks in the same order, avoiding PostgreSQL deadlocks under load. --- litellm/proxy/db/db_spend_update_writer.py | 48 +++--- litellm/proxy/utils.py | 8 +- .../proxy/db/test_db_spend_update_writer.py | 143 ++++++++++++++++++ 3 files changed, 174 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index c06e1850d9f..418fc2d2f02 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1133,10 +1133,12 @@ class DBSpendUpdateWriter: timeout=timedelta(seconds=60) ) as transaction: async with transaction.batch_() as batcher: - for ( - user_id, - response_cost, - ) in user_list_transactions.items(): + # Sort by ID for consistent lock ordering across pods to prevent deadlocks. + # batch_() issues statements sequentially within the tx, so iteration + # order = lock acquisition order. + for user_id, response_cost in sorted( + user_list_transactions.items() + ): batcher.litellm_usertable.update_many( where={"user_id": user_id}, data={"spend": {"increment": response_cost}}, @@ -1188,10 +1190,10 @@ class DBSpendUpdateWriter: timeout=timedelta(seconds=60) ) as transaction: async with transaction.batch_() as batcher: - for ( - token, - response_cost, - ) in key_list_transactions.items(): + # Sort by token for consistent lock ordering across pods to prevent deadlocks. + for token, response_cost in sorted( + key_list_transactions.items() + ): batcher.litellm_verificationtoken.update_many( # 'update_many' prevents error from being raised if no row exists where={"token": token}, data={ @@ -1232,10 +1234,10 @@ class DBSpendUpdateWriter: timeout=timedelta(seconds=60) ) as transaction: async with transaction.batch_() as batcher: - for ( - team_id, - response_cost, - ) in team_list_transactions.items(): + # Sort by team_id for consistent lock ordering across pods to prevent deadlocks. + for team_id, response_cost in sorted( + team_list_transactions.items() + ): verbose_proxy_logger.debug( "Updating spend for team id={} by {}".format( team_id, response_cost @@ -1290,10 +1292,11 @@ class DBSpendUpdateWriter: timeout=timedelta(seconds=60) ) as transaction: async with transaction.batch_() as batcher: - for ( - key, - response_cost, - ) in team_member_list_transactions.items(): + # Sort by composite key for consistent lock ordering across pods to prevent deadlocks. + # Key format "team_id::::user_id::" makes the string sort equivalent to sorting by (team_id, user_id). + for key, response_cost in sorted( + team_member_list_transactions.items() + ): # key is "team_id::::user_id::" team_id = key.split("::")[1] user_id = key.split("::")[3] @@ -1350,10 +1353,10 @@ class DBSpendUpdateWriter: timeout=timedelta(seconds=60) ) as transaction: async with transaction.batch_() as batcher: - for ( - org_id, - response_cost, - ) in org_list_transactions.items(): + # Sort by org_id for consistent lock ordering across pods to prevent deadlocks. + for org_id, response_cost in sorted( + org_list_transactions.items() + ): batcher.litellm_organizationtable.update_many( # 'update_many' prevents error from being raised if no row exists where={"organization_id": org_id}, data={"spend": {"increment": response_cost}}, @@ -1441,7 +1444,10 @@ class DBSpendUpdateWriter: timeout=timedelta(seconds=60) ) as transaction: async with transaction.batch_() as batcher: - for entity_id, response_cost in transactions.items(): + # Sort by entity_id for consistent lock ordering across pods to prevent deadlocks. + for entity_id, response_cost in sorted( + transactions.items() + ): verbose_proxy_logger.debug( f"Updating spend for {entity_name} {where_field}={entity_id} by {response_cost}" ) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d2dfa177515..800a4be37a8 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -4874,10 +4874,10 @@ class ProxyUpdateSpend: timeout=timedelta(seconds=60) ) as transaction: async with transaction.batch_() as batcher: - for ( - end_user_id, - response_cost, - ) in end_user_list_transactions.items(): + # Sort by end_user_id for consistent lock ordering across pods to prevent deadlocks. + for end_user_id, response_cost in sorted( + end_user_list_transactions.items() + ): if litellm.max_end_user_budget is not None: pass batcher.litellm_endusertable.upsert( diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index 4d584349342..9d4d3c4a560 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -1508,3 +1508,146 @@ async def test_commit_spend_updates_uses_pipeline(): mock_redis_update_buffer.get_all_daily_end_user_spend_update_transactions_from_redis_buffer.assert_not_called() mock_redis_update_buffer.get_all_daily_agent_spend_update_transactions_from_redis_buffer.assert_not_called() mock_redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer.assert_not_called() + + +@pytest.mark.parametrize( + "bucket_name,input_dict,table_attr,method_name,where_key,expected_order", + [ + pytest.param( + "user_list_transactions", + {"user_c": 0.1, "user_a": 0.2, "user_b": 0.3}, + "litellm_usertable", + "update_many", + "user_id", + ["user_a", "user_b", "user_c"], + id="user", + ), + pytest.param( + "key_list_transactions", + {"tok_c": 0.1, "tok_a": 0.2, "tok_b": 0.3}, + "litellm_verificationtoken", + "update_many", + "token", + ["tok_a", "tok_b", "tok_c"], + id="key", + ), + pytest.param( + "team_list_transactions", + {"team_c": 0.1, "team_a": 0.2, "team_b": 0.3}, + "litellm_teamtable", + "update_many", + "team_id", + ["team_a", "team_b", "team_c"], + id="team", + ), + pytest.param( + "team_member_list_transactions", + { + "team_id::team_c::user_id::user_x": 0.1, + "team_id::team_a::user_id::user_x": 0.2, + "team_id::team_b::user_id::user_x": 0.3, + }, + "litellm_teammembership", + "update_many", + "team_id", + ["team_a", "team_b", "team_c"], + id="team_member", + ), + pytest.param( + "org_list_transactions", + {"org_c": 0.1, "org_a": 0.2, "org_b": 0.3}, + "litellm_organizationtable", + "update_many", + "organization_id", + ["org_a", "org_b", "org_c"], + id="org", + ), + pytest.param( + "end_user_list_transactions", + {"eu_c": 0.1, "eu_a": 0.2, "eu_b": 0.3}, + "litellm_endusertable", + "upsert", + "user_id", + ["eu_a", "eu_b", "eu_c"], + id="end_user", + ), + pytest.param( + "tag_list_transactions", + {"prod": 0.1, "customer-x": 0.2, "test": 0.3}, + "litellm_tagtable", + "update_many", + "tag_name", + ["customer-x", "prod", "test"], + id="tag", + ), + pytest.param( + "agent_list_transactions", + {"agent_c": 0.1, "agent_a": 0.2, "agent_b": 0.3}, + "litellm_agentstable", + "update_many", + "agent_id", + ["agent_a", "agent_b", "agent_c"], + id="agent", + ), + ], +) +@pytest.mark.asyncio +async def test_commit_spend_updates_iterates_in_sorted_order( + bucket_name, input_dict, table_attr, method_name, where_key, expected_order +): + """ + Every spend-bucket code path in _commit_spend_updates_to_db must iterate + in sorted order so concurrent pods acquire row locks in the same order + and avoid PostgreSQL deadlocks. Covers the 5 direct loops (user/key/team/ + team_member/org), the end_user path in ProxyUpdateSpend.update_end_user_spend, + and the shared _update_entity_spend_in_db helper (tag, agent). + """ + db_writer = DBSpendUpdateWriter() + + captured_where_values = [] + + def capture(*, where, data): + captured_where_values.append(where[where_key]) + + mock_batcher = MagicMock() + table_mock = MagicMock() + setattr(table_mock, method_name, MagicMock(side_effect=capture)) + setattr(mock_batcher, table_attr, table_mock) + + mock_transaction = AsyncMock() + mock_transaction.__aenter__ = AsyncMock(return_value=mock_transaction) + mock_transaction.__aexit__ = AsyncMock(return_value=False) + mock_transaction.batch_ = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=mock_batcher), + __aexit__=AsyncMock(return_value=False), + ) + ) + + mock_prisma_client = MagicMock() + mock_prisma_client.db = MagicMock() + mock_prisma_client.db.tx = MagicMock(return_value=mock_transaction) + + mock_proxy_logging = MagicMock() + mock_proxy_logging.call_details = {} + + buckets = { + "user_list_transactions": {}, + "end_user_list_transactions": {}, + "key_list_transactions": {}, + "team_list_transactions": {}, + "team_member_list_transactions": {}, + "org_list_transactions": {}, + "tag_list_transactions": {}, + "agent_list_transactions": {}, + } + buckets[bucket_name] = input_dict + + await db_writer._commit_spend_updates_to_db( + prisma_client=mock_prisma_client, + n_retry_times=3, + proxy_logging_obj=mock_proxy_logging, + db_spend_update_transactions=buckets, + ) + + assert captured_where_values == expected_order From 2993e45ad18e7508d7f4a262608006bc787082c5 Mon Sep 17 00:00:00 2001 From: Michael Riad Zaky Date: Tue, 5 May 2026 11:37:12 -0700 Subject: [PATCH 16/85] allow non-admin roles on /compliance/* read routes --- litellm/proxy/_types.py | 10 +++++- .../proxy/auth/test_route_checks.py | 33 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c6653a722d6..7a049dcc5de 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -656,6 +656,13 @@ class LiteLLMRoutes(enum.Enum): "/health/services", ] + info_routes + # Stateless validators on caller-supplied log data; source logs are + # already accessible via spend_tracking_routes, so no scope expansion. + compliance_check_routes = [ + "/compliance/eu-ai-act", + "/compliance/gdpr", + ] + # Routes in `global_spend_tracking_routes` return proxy-wide spend across # every team, customer, and api_key. They are intentionally NOT included # here — non-admin roles must not see other tenants' spend. Admin roles go @@ -675,9 +682,10 @@ class LiteLLMRoutes(enum.Enum): ] + spend_tracking_routes + key_management_routes + + compliance_check_routes ) - internal_user_view_only_routes = spend_tracking_routes + internal_user_view_only_routes = spend_tracking_routes + compliance_check_routes self_managed_routes = [ "/team/member_add", diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index cf6feabf85f..3e0b1b739ec 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -53,6 +53,39 @@ def test_non_admin_config_update_route_rejected(): assert "Your role=internal_user" in str(exc_info.value) +@pytest.mark.parametrize( + "role", + [ + LitellmUserRoles.INTERNAL_USER.value, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, + ], +) +@pytest.mark.parametrize( + "route", + ["/compliance/eu-ai-act", "/compliance/gdpr"], +) +def test_compliance_routes_open_to_non_admin_roles(role, route): + """Compliance routes are stateless validators on caller-supplied log data + — both non-admin internal_user roles can call them.""" + user_obj = LiteLLM_UserTable( + user_id="test_user", + user_email="test@example.com", + user_role=role, + ) + valid_token = UserAPIKeyAuth(user_id="test_user", user_role=role) + request = MagicMock(spec=Request) + request.query_params = {} + + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=role, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + + def test_proxy_admin_viewer_config_update_route_rejected(): """Test that proxy admin viewer users are rejected when trying to call /config/update""" From 85d4d96c1bf1d823b1c66d5464768d568302b3f0 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Wed, 6 May 2026 00:28:42 +0200 Subject: [PATCH 17/85] fix(proxy): preserve HTTP operations when injecting WebSocket stubs into OpenAPI schema --- litellm/proxy/proxy_server.py | 88 ++++++++------ .../proxy/test_openapi_schema_validation.py | 107 ++++++++++++++++++ 2 files changed, 158 insertions(+), 37 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a5905765c6e..b136464fd2d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1057,6 +1057,52 @@ vertex_live_passthrough_vertex_base = VertexBase() from fastapi.routing import APIWebSocketRoute +def _inject_websocket_stubs_into_openapi_schema( + openapi_schema: dict, websocket_routes: list +) -> dict: + """ + Add a synthetic GET stub for each WebSocket route so it appears in Swagger UI. + + Merges into any existing path entry rather than replacing it — a WebSocket route + that shares its path with an HTTP route must not erase the HTTP operation. If + a "get" operation is already documented on the path, the WebSocket stub is + skipped to preserve the real GET. + """ + for route in websocket_routes: + base_path = route.path.split("{")[0].rstrip("?") + + parameters = [] + try: + if hasattr(route, "dependant") and route.dependant is not None: + # Handle both FastAPI <0.120 and >=0.120 + query_params = getattr(route.dependant, "query_params", []) + if query_params: + for param in query_params: + parameters.append( + { + "name": param.name, + "in": "query", + "required": param.required, + "schema": {"type": "string"}, + } + ) + except (AttributeError, TypeError): + pass + + path_entry = openapi_schema["paths"].setdefault(base_path, {}) + if "get" not in path_entry: + path_entry["get"] = { + "summary": f"WebSocket: {route.name or base_path}", + "description": "WebSocket connection endpoint", + "operationId": f"websocket_{route.name or base_path.replace('/', '_')}", + "parameters": parameters, + "responses": {"101": {"description": "WebSocket Protocol Switched"}}, + "tags": ["WebSocket"], + } + + return openapi_schema + + def get_openapi_schema(): if app.openapi_schema: return app.openapi_schema @@ -1079,43 +1125,11 @@ def get_openapi_schema(): route for route in app.routes if isinstance(route, APIWebSocketRoute) ] - # Add each WebSocket route to the schema - for route in websocket_routes: - # Get the base path without query parameters - base_path = route.path.split("{")[0].rstrip("?") - - # Extract parameters from the route - parameters = [] - try: - if hasattr(route, "dependant") and route.dependant is not None: - # Handle both FastAPI <0.120 and >=0.120 - query_params = getattr(route.dependant, "query_params", []) - if query_params: - for param in query_params: - parameters.append( - { - "name": param.name, - "in": "query", - "required": param.required, - "schema": { - "type": "string" - }, # You can make this more specific if needed - } - ) - except (AttributeError, TypeError): - # If we can't access query_params, continue without them - pass - - openapi_schema["paths"][base_path] = { - "get": { - "summary": f"WebSocket: {route.name or base_path}", - "description": "WebSocket connection endpoint", - "operationId": f"websocket_{route.name or base_path.replace('/', '_')}", - "parameters": parameters, - "responses": {"101": {"description": "WebSocket Protocol Switched"}}, - "tags": ["WebSocket"], - } - } + # Add a synthetic GET stub for each so they render in Swagger UI, + # without clobbering existing HTTP operations on the same path. + openapi_schema = _inject_websocket_stubs_into_openapi_schema( + openapi_schema, websocket_routes + ) # Add LLM API request schema bodies for documentation from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec diff --git a/tests/test_litellm/proxy/test_openapi_schema_validation.py b/tests/test_litellm/proxy/test_openapi_schema_validation.py index 68d537b2593..b44edc8a3bc 100644 --- a/tests/test_litellm/proxy/test_openapi_schema_validation.py +++ b/tests/test_litellm/proxy/test_openapi_schema_validation.py @@ -140,3 +140,110 @@ class TestCredentialEndpointsOpenAPISchema: assert ( "credential_name" in sig.parameters ), "get_credential_by_name must have a credential_name parameter" + + +class TestWebSocketStubInjection: + """ + Regression test for the v1.82.3 bug where adding a WebSocket route on a path + that already had an HTTP route silently dropped the HTTP operation from the + OpenAPI schema. + + Related case: 2026-05-05-madhu-swagger-responses-missing + """ + + def _make_fake_ws_route(self, path: str, name: str = "fake_ws"): + """Minimal stand-in for fastapi.routing.APIWebSocketRoute for the helper's purposes.""" + from types import SimpleNamespace + + return SimpleNamespace(path=path, name=name, dependant=None) + + def test_websocket_stub_does_not_clobber_existing_post(self): + """ + When a WebSocket route shares its path with an existing POST operation, + the POST must survive — the WebSocket stub is added alongside, not on top. + """ + from litellm.proxy.proxy_server import ( + _inject_websocket_stubs_into_openapi_schema, + ) + + schema = { + "paths": { + "/v1/responses": { + "post": {"summary": "responses_api", "operationId": "responses_api"} + } + } + } + ws_routes = [self._make_fake_ws_route("/v1/responses", name="responses_ws")] + + result = _inject_websocket_stubs_into_openapi_schema(schema, ws_routes) + + assert ( + "post" in result["paths"]["/v1/responses"] + ), "POST operation must be preserved when a WebSocket route shares the path" + assert ( + result["paths"]["/v1/responses"]["post"]["operationId"] == "responses_api" + ) + assert ( + "get" in result["paths"]["/v1/responses"] + ), "WebSocket stub should also be added under 'get'" + assert result["paths"]["/v1/responses"]["get"]["tags"] == ["WebSocket"] + + def test_websocket_stub_added_when_path_is_new(self): + """ + When a WebSocket route's path is not already in the schema, the stub + creates a fresh entry — preserving the original behavior for WebSocket-only + paths. + """ + from litellm.proxy.proxy_server import ( + _inject_websocket_stubs_into_openapi_schema, + ) + + schema = {"paths": {}} + ws_routes = [self._make_fake_ws_route("/ws_only", name="ws_only")] + + result = _inject_websocket_stubs_into_openapi_schema(schema, ws_routes) + + assert "/ws_only" in result["paths"] + assert "get" in result["paths"]["/ws_only"] + assert result["paths"]["/ws_only"]["get"]["tags"] == ["WebSocket"] + + def test_websocket_stub_skipped_when_existing_get(self): + """ + If a real GET is already documented on the path, the WebSocket stub is + skipped — a real operation always wins over the synthetic stub. This + closes the same trap for future GET-vs-WebSocket collisions. + """ + from litellm.proxy.proxy_server import ( + _inject_websocket_stubs_into_openapi_schema, + ) + + schema = { + "paths": { + "/health": { + "get": {"summary": "health_check", "operationId": "real_get"} + } + } + } + ws_routes = [self._make_fake_ws_route("/health", name="health_ws")] + + result = _inject_websocket_stubs_into_openapi_schema(schema, ws_routes) + + assert ( + result["paths"]["/health"]["get"]["operationId"] == "real_get" + ), "Real GET must take precedence over WebSocket stub" + + def test_responses_post_routes_registered_on_router(self): + """ + Sanity check: the three POST routes for the responses API are still wired + on the responses router. Guards against accidental removal at the source. + """ + from litellm.proxy.response_api_endpoints.endpoints import router + + post_paths = { + route.path + for route in router.routes + if hasattr(route, "methods") + and "POST" in (route.methods or set()) + and route.path in {"/v1/responses", "/responses", "/openai/v1/responses"} + } + assert post_paths == {"/v1/responses", "/responses", "/openai/v1/responses"} From 062b5b31fb0d7e9f7cce989d22bf0c0561c83ea9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 6 May 2026 00:26:17 +0000 Subject: [PATCH 18/85] Add main module header comment Co-authored-by: ishaan-berri --- .evidence/main_header_repro.png | Bin 0 -> 61972 bytes litellm/main.py | 2 ++ tests/test_litellm/test_main_module_header.py | 13 +++++++++++++ 3 files changed, 15 insertions(+) create mode 100644 .evidence/main_header_repro.png create mode 100644 tests/test_litellm/test_main_module_header.py diff --git a/.evidence/main_header_repro.png b/.evidence/main_header_repro.png new file mode 100644 index 0000000000000000000000000000000000000000..d71ab04d02a56b3ad8f2e8015bf53ca13f8cd8d3 GIT binary patch literal 61972 zcmeFYbyQUE-!6;^h_ryTv`9&p4ALbaInvT89nuOT4TE$e-3;9!Al(fTLw5}gL%iGg zKIi$JC*J3*b^biFX1T`Ap4oSN>b|b)6Zlp^3iA=kBNP-AOc`l$WfYVL=_n|7U!vUw zt{_i?ZBS4UGcw{Lsvnbf=3KP#*C|mC7_7dsxTD{_>vPXIFTf|DrV^`SLc?-9e?n+0 zcf77reLLUDa&F@Db82c1)$b3m?tkf_NVh@@zq;W|)a$9K4~O6i@5i#~#r8T3dZyY9 z(qYF6hSgQwxdS}$*URi;B|+fNN8sP??_8*V|I>x`#^LYBPa*&B^6#biKU|Z5j<~g_ z<=oN|VmY|Zw0I5-`_r*yh3=27w_g8U+1DH-n-*ADIp(aS>{jp(r&)zq*B!jw^Q#2W zAB3c^bUfXDmd1{THIq~#5{lNPTNk-xUK*6`3RwVm@~k;?Pb4~>CV4V$k2VL#|GKSi zD=N?sdutBMv$MY`YL&x8B(&%uGl&mqkeyb$J6f6-Ltm+HJ!`cT+4!7bK;)@XUIhtXm>%P0Bhc&t?F1sUvh&# zObA}3r6<}I96rh~qgWRfH?BGJI=rWApgkYC08U#k!!t-=RsZwMaFLOOwFTRihit-o zTprmmN`kG66%#xqCuICf>%q!Db&gcqS={ZAuwY>!(`-8dy>TA9f5x(H4)|Zr-Kk&i3r~()s)lVt}p15J*vI zO&Tr!ax4^-PazQ#T>iybzNNnI9l@wlIWE1fM>}J)^So6GgI?slAk=LNUsJuFGTXg% zj!S>@PCZYDS1s#ExawGCo(n%&Z6GqE(;BW?x#nds)b0qdYSj@5r+wS_Wwz#1 z{Go-Te2ntj+D1!VSA>*~9O~9OzXr@u?@LPgzF<|T1gdC&(_ork)xfZ6b*xt(ZaMRx z|m=_Hy%?Dp;+o5B_5~BLhymh2c6=Jg>`Hb>AxP zO(PvFm&;)H!B#Inm3Z6tS=X^wAMm%o_m93?d-g^~t*x*;M*^G*CAP^GFgN;1KM{7# z@YD5zsleQ;&!e}XSVJNNeQFs8OKO9VR;hEc+JSO|-Swo>N$(`CWJ&BYF$a8WX1n%x zMkik8F=35433%Zv9_JgG0$Di&o?Wexw<0R zCQC7P8#*AAH;&a9g0NvoI8h4&Iu(QI87<^#)jp4f%ImI_2f(BYIlX5<89rXg(Rk#U z=0^MmWXmQW=($kL5e(Tbt_y!|toXa611rL;}sdq>vxHcO`fe*f9xlw8Rwi{ikH<2Qy zA1VBIHH6IN9ij-Dnj76yCi_0D=`^ZxJMCVED0qD=f5rsta0)89;^W&yoWq;Bf+5df zDH|-V{T&(LJ04vMfmlFD;W(TcY}k4s8t`FXX>%|Gl-aDG>MgoaJE>2EPE~8OtUC(= z;>l+}5v{F5obrK2bSkRm!9h<_DKwd24Ct$-CV0b~NwPbJqQlV2TYP^S2H0~3wju$T zNNuG_wUWbb3SB>PwWy=_;YNrs{H(>cni&`57wGDmmmeKkTGZ1Lf5ah(D1Bdh?ny``u9U@n zqAWjIQ$Y}<*MN?d0UVvH5lwY*IYg{de_@@(%+yI*#&P275VNV-^?Q}v=T zkOwGDa%RP-82G}9kU^_{fGz9z4;DUWM4Niaf3X<)kQRqHRZmGdF8O?8IBU+_4!3n5 z@EUrEt}SWxM*D6vZ3)eviG8Y*Mtm-VgqQg=5Vhn4!A(>2cvePw*aXf!Ho1^KSP zHpR9ghgy?V1#P@V1Cl2<4ws-ZN|WL}h56~KC0Co(w4xW2yKuNcK$Mv4u>X&P-ZMpO zv#jBck;-Oi5uQH()p$}kA%NBih z9 z%;75>9G>ctVdcOZ;L%!^^pKQ-Qdd_c5LIUK%xg6kPo?1S%VSomAC_x8I&xAH1Zw!5 z6mtxPIEAKdHjAHG#hORitJU5rD-92o?(vUZ_tHVa&u$x@aK}9?D9-or+#hMD^iNIN zMNpqdMbCWy+N^zJp-4NUjMd(j$^5p9;d*A%lR4;-AhlKmeNn-MB-g zFET-mGu|zrmfj@Z!TH%{)}(JMV$QOCMb6guKqd{{GH#lNN3BU?K7B^z11T!l!Wk1} zNn>dZFH`%A@%K{pUzsH|spXW;_Q&tl^=-98t>6vNm~5kuFYX*ir)7TkZQ7$rRf zn1Z`^amcHxfeyns!`ndn!yrghVAS7Hev6Ns7$>u#oNXv|as!^dbj2K&2_jI%wv8V4)~-onLyNl&$M(kSO-HMovyeL;Wis;c&- z3qDoAR2!=fb)#@>I5xzB)s?+u;9k2@*g7L;o}1!eIbAAlh+3a5TLR2$IuI&$#!_45 zWe{5L&Xv8@69je{NYN9hX>$L@LH~D%W>tJ!KOc_FcTSO4v%wH-OkMcqOhEF!CM0jD z@)DKpW4Vm-jUg{}JP~9s@9gZWHh`f2V>^jYD4#QZUq@s*Uf-2cnD6oYmL+%F2Of@F z{(HxHA9n{ADiuVLR=8h(e;4jDpkQ&)z3O=!__tbqy$0OS8UNrqGuBmO5y=pH)>upV ztWoFh_g@DOJw5v;J)xN0A8{M`RqFqGy{2jVKdP+FLd?T(X{5-7F2HZ4&1}u$VqM^p zQ#4$ZM{8`4u`n}kSl(4+HW`JH3qJ7o!zAOXLEL*!OjSbF=p0qUeq-n>*4Ee8uaRv7 z@mgBiA|xYOo34j)alj&Tqo0Ye6+F&}Xj2E=3rw-;)qclvM0IMF&2oT>;C1GkSV@d;5Mw4=xf5YlLDyvk`;% zx#$u0YmZbh&O2R?*`f4g`(<}_r8Pw%cvLt3<0dzDXATGP$7(3=Wkh}o+S)lVK$zor zdH1GOD4-0xb0Q8$+owg%)^J%gbYN~Wgp_A}IZ)Hc``L2c1|-Tu5!C4+mknr3@cR&t2e^fxS}Qsw42%-$sPG)fJokR&9y*49@0+K>x7SLByP;Ed#}6%>5tg_85{%s19IHMP`L zT8@p46ESNiMJASCb`ogB$A{ojfV^7!3d7)ZVPrg+-@bj)RDY{_Oer|fIXT%ilv`87 zM}0Nlat|3C6m&mue0-eYO~Oc9o0yao1B0L-2glNJH~yC|Uv^wXQpJLw-<4}g)Bk+u z&Py2UocJEyQr(GVab2Ke?cX|2#kdWi= zpFYVE4Pe~11T~T8-O=COQj?XHjm&=YA)^4fg2-SY<@Go}_zZgW)hCmf9gT`UJDbyV zaC5djFE@ANAl2K<+(JH;-J$0POc~yYANF#``gye1!nc+qB9;_hQ)iD^&c*j;0(iZ} z?KU&=m_!dA{63C5@;A)uEsVameOrDth97Zy0sn}yVVLwpv0JjI>+I=)TbBVD?jJ`dBZus&;oC5eSLF& z`{Spst{13h8K-;mbV>KFKZ)txZ*{CGy(*FQWk>BmV;m7I$0?Qa;<+B`n)-MJ2^?T$ zO-Rrr-rRH%+fM>HMf9#d!V4G;3!xEC%n0-%I%dE^)C-_+f%5eFTRJ93o5`1jV>Bsv zZwkI8@&fCmqN+MuXJ{(O#AMN!_I3Ci9lw93>e_rijyQ@C`mhk5ay7 z)){zK{)iJcp8_SB@>D zOe_En*Y0FzpWmpfBG)@KbYQaJ#N%!%bLDq>a)dwoWvm{isD%-%0at+K=i@1imIzoS zDWWOUdEx$`ydXxVeaTvd)q89rb zl+@Loot|aE)s>X=^{1^7WyZd!*!<4t;R`m5$Gg%9#BQ_x_23x|jr<#V z?;B%Z_cMz8{QRt}EIf2z|DW=^g@lDE7>S>!=rl)1MV((_o?EXmD_I7n{Cf+yi=-ao zwJsNdt4)zvvKAKuQS^AhE2`;pON24gjv(Lh&L*xGTS$qZ-4KyTUq+q;**>HyC9xZ! zU`8r%;P(ATw~EL0)#=R*<0r4CYjmp5lkdj;&ZGOXW1eH~nJ;;W20)?P%ex=mzmH2! zrfZlV7)Zgvqa>1Qd~$RphufT%tfTF%QGM#A8r76?lb+RUwY$@Bc{y9>Di;39Gv2rw;EON``Q5`>5Bt2D2$|O5UddzASBfkcZ#NSyOi&Np`M>)mfBn2()zuTB)Yc3WFp3kwSkrLb~xvT#M~oE)ei zR&x#PjO?X_g*%80Db%xl28g4BGp-f#T*=B@raNQmsc2UlsuCR?9l{Qw`ST|pdS7q7 z4Un4h33_##6&CDk`KPC6s;wDW810eptQ11 zY15y;{gmmmC+-$$b$jR9yL#u{t>o|-z(g3ky9HgWXnsvHd)~ldDcl)%gWG>Od1%6c zlYbulgV=hrH!z^#$(+=NhF6sxos(zNqYQa@g*P0`UUcQKtcPb@c0L{ylu~G=<<$(3 z$CL;OpsevVl(N8tcDkP&K0ror;5Db+gP}t1qsGQWWoFWv1qn&L~v$&|gK z4&XoJJQdT?5kl<@W7BJzD!EyYidqReJggWS8_iJw)p$2c(i()-aozLzU(TLzBgM43 zy80D&v2t>Ia~rIt-XtT7SLk}N1Yi_;x3|3bX=`tLdwajc^hUkY!xR}D>wBXivD$M& z%002YwW*BZL1D9g`r(L>dA8Ej#zXt6-i48Uc(|6I|37Yx;T@q@LjKUF=Efh2va=<1 zN@dW@$Md75-DRXdGQJz=?WKX_21MNR3_qsvYfAA_H!=A-q^E(e+ZeIKpd;hderXXX ze{$e#&0KGk9M5U24u;tTK-?&m4;4r z2MlvNK6G?)fCh!w?o4%0O|5J!iZMIHNYj-U{$2aXC{fz#5dfx+7t++4UBSfuOtKB|Fw_t~YqI=sxR0K6^G z>)_Mi`luB3TCSY%vjE@KK(VW0ER@i^P1ssj_W&WIP%0W`h6jiWa=Ynm@j&mVB6BSG z{C4*3LD93@uQUTmHw6t1RznMEor=kO`@F9+XMYT)-J#WjtjW}b;0+zRlid4x_w;DD z*<`nlRY75EYs*LR2!hiBSJHUFhPn6lxfKT(f{LaBLdTxzWN zxdjXQVXP|W@Q&!~F*~jMm=?wzVXF0{(xqsaBU5em*C(^uVP|z{>p1xM$?tD@mC zVzxn|7f0t+A$yK5s$vGAMsudh!bE`p-FCK) z6=hWlvTrgcrkplj7sW2jyEa&e2Kb~K>{fG(i~?9u)~(s)#b!iA#6d|3@XqkZE}AJp z)N7R6%%d6AV#M%s-BOn*X)34x)U4E>3DT3~xf6g-l@T-Y#);&mnXS3`X13l$r;xj* zs&?Nx;vqJEs0e zRHsz`>5-`weTVFOJ;-IpZil|vx<3P>jQ;-pyY1-2E=A)j zC#RFBs3Au-cMC<5 zILfeJC!qxva;bRc7vvuv9m)0fjA>S%9vmm}nFsxFp`yRUrQxBMl#wxSS)zOR&=0U` z8YyTVB2jCLs8!4#T2fM?`eE*;&G+xRjjy(6l{~K@o?xq|JQt(9_=a?V&qr{W9lU4T zZvt{QAz_Szcbp=c>(%oZH#ubntlwuYJdx3%sPEf%$oOlGnbh<1^5z~rGDs~Kf1yEL zNqfTpx%|=jtDX-bS-w5Exh^X!JFlsM(aTdqUO|InRUr{eytdz}q0sz%3!uuXJtD#p zHOQbll7VG~n2RTv68`r8xY$_aE6~7vQ(GAU6ALpl5H{rBypcpA>+5QRb(n5j_?~gq z{L9Yx`1(kCu;+Wa&s?M4J}G>RjZt7%?ZIwrYMhmmN_BWV$;eB1_xvd_`fHkz^V~o{bi?;JXw;6;)*1BU|i8PWOwI5tv zU46}v9E=l^3OBh)ZwKsv9->`u|D_H){@ged`*r61B{Rl;)tm!&7eS}J2FpR+ExOdf zX19y|v^FU;yb>#^ymU%-R-J~IAG_W{byENv$fFjQgkVtUOQUrU4no~e3k`D=P@!aW z0AvH!Yj$?(Dp{;_QPmUHhZB(cv3qxz7!=+&C;B|(OmI4Q7!n$)QEE}Su(~~oe%1=Q z))e`dJ-RyE-H0pNw^ePR4aPCMfdwiBR7JC94xgvfS~JZ^PZdOUm)TB*y&Bl z>`a1!OlHdE*3st19h`e^RLjgPF>=1atJ3HOl4zbwfFl~a> z)%~Nxst@zWuZq{AXA-?M&VUj0^zDS`nXTp zBOVRl8e}#IyRy2}Kb)!N%vP*-r8)4_)ipL=t@pmUFuZNb?LuSU%D@V_C+dr|SoT;( z`6St)G+FF=>k3d+CSBz&scOPS*LKq*NTsjx5;*2X$d;t*;oFT)JeSF`nG0IPi$J!=`cfdpBP?sKJ@pCk5A-nZ4#&3 zrOLANW2U3aiZmTiNIsR{t)3{pHJ*V9X4Sd;8tA!|j7vgWldo@u@EhJ%*4ICyXQ@d_ zCGIoQ=M?Lh_Fv5Z_#v)P{F%o0oc~i?f}#{TcZZ&i{Pk@7XeC_0ZR2gufDSE3no~)`V3=fIpyLQVYT)AwKm64m;qr+dw()c+57p~8dF8rS1DZfxooX^)i&2oNOe(Z z0F=CU%4C>1M?jheunJ62VnNo8wb1nR^q>1pu93$ph~>aY1}O+U(&Fi_;T{mT>S^Cu zz@Ou(1EPdt>MQFhGJkP$MLAvsUj*6)-sUrMWCN zHa0nVa?;LDS69F&gW?lH!ZNsor)H&v6ZgSdDUJVJz02ez@4IT}(1f~6pD)1<_73W| zHPlR;k+N%Jl`|!aG*-L2+}zxp93#LPja7$_?dtl&zan~&GXT7;(m#r+EC4eyGRh}# zRE8Gs0pLNE)$GAy8|vl3^OQzp;idU5v^PdmLkmhGirQCKr&Y9~gW>+(=4DmJ-LN1U)H!M8Q7tt0j;8|;UNhuC{kyK{{Nl2|9> zRH>bY%y-j-W^<{>u&l0srtU#zO;qml@V}Il2GCxN$7oYk#wxqjPo*Ba{kN#2VZ3l+ zv6+$8E3Z~@Qz9aH`J#~Hs2RBJf#6jYZ$+yqvL??6>>J76M1%{t4H$7jy^p!E2Avx z>h3N=8fTOTW1J|cV> z1wn(imsV8W8?$3#($W_Dzi(S{=!^BK(?iZLFVL;k7Um{%ioUcL+T;MKe_UF<9_s;o z#Jq%mHi4f#_*+_9>2md)fPer5hkzhjLM}!^LNXvh*brP%QgP)pxo=@%G1gl}NQlp% zC?lhQRZpr(lj2uNljr^VeK`w_ghYFhpmSAuS-LI>32|uZ%i)F(GpXuDPZT<)W>!~L z1o-$s?w1e~y^D*B>1mDRFH?zaB-bW;NILccXPX(b3cBos%+4V zAqn7j1>oYio86EWG5-`nhqWK2G^Ev5A8ah;8hig+=xi|OXx&Q@&2DFEb5f^%dnOoa za|>*06D6^^fyLz!8|iHQ*;_R=78Vv`;>_TJ?aNAGp&H68oA)I~(~kgxM9AF@0>_|% z+}Fhh5Xm4|DtsO;yxW)xaJkY8+*N8+06LOK7gU!}7aMy9e7U_tydXdS@)C0a@iF9r zQ&O4@1S*DBSAYJj@9Z3N;3g>OE-LMzO?DE|L_U4Y_1gn?mM(kiD$g>~k57(?SaeJ* zP1naK*C)J9eYOh4cqG0F&!$s?zU_@2Ea!9Y3AsH!Ju;@``67>Hg+)jbLi4ckYewhB z#s*OD?oXd+$;br4;GgzQ2JjfYd2|r|4=~2Z#zcL56_fc=;-m9&a@L1?%hS^vBwu1$ zONk0TE7{CFzCfN|0vjzW`vw5w4Ba#nk{hohR+-#@T(z)puyo;&X7spG2mbxFni{u| zP-Du(IW3LQTuUj9=WmxoQR1F%b-vvlWV`#x)IVq|io=XT6U1fD4o_{E*- zva*JB>eUbS0H_PpsFISGUyjxR-)C%jN+BG?!oukKgj5X8_54{rP?3RHb>*}Y-FVs9 zT)JB?l)mChNJwb2o;kG^9GMP0fBr(VsvOUpt=fA2F*+723yZwRwZO|~uN4#qy1L@7 z*CvZ~8u9JxD=N6?_+8h-K0v~)2L{HpF81M45|YO!n+$AhibV*-iRbCDjKzxrVGMj3 zRSu-)2=B|O%A-j^s%i`j48Tjn$oZ2}V(53#Qad|wMR#^~(W%;`Jb^;398D)VxmJ5GMxGS- z*`S~x)+9kvQhXNWn-Co(FnAC^(tBgs0P29Bj;=Pn;S~$_2>={o@4;cXZ+vII4q>4! zZBO)n!6|b(>d5@@1336~LMg~+n^SQ{`lluMhz)HfM@Fb2PdKhWmXv&Z3xLmRYHG@e zUsWGUIQ`XdP+oj|Sx`^(Hn(@xw}KowmIK#@Q%(+!$v_5~(Up}62_#Nmc&IYf3e>50>8KX7-t)7uL2=TlWwFp< zH_fj909-_c=rwNR8L>G9iSDXK5W`M|jt+V$Y)XUFTJab0#w^jXu~L$d;HdI*TmZ&< z!NyiyzHiHWtiVeYJ2IjS<^`734nQ|-A|il8fp};R==b6-jx#Nm6ciMkd?85~DG{J$ z#;RUZ?4#b;-ZV8fAw<{Fk2=_jK6ny%P7_fYh)zX&*>Vq*sz~{3^cv~<2n3~BW!3lYIvUDZP{%fs1WpdRs6m>COd9Tr(tB2oq;hHMk zs_E+Tm>mo)=O~<=kv{8AMqI4Gv8TrvK7dl$7|!w(E-y~?%AIwH7#N(KoE8VSedHM4 z{RX+V#}qf>g|A2P=3mj%3otPS=M%zju7e+8?9VkaDFdttU~)abz_yox;$6!K-KLNF zrZ%u|--c>~M*)mmXLc5is>=BkVK+!ZsF+MU-;-Gk^TsT0yF{b3W{Ii@ein)=Q}zq3 zGW@}CyBQSH*toDPB^prY_t3M7goH$!73@00iaz zxH05^J0sPM7D1wpBSI2Kjgk8@Pyy3MU8@W{kz``y?_sxp6^( zY`3s*Wp&lTLZ?Dh#sByng>}D(xlEwNwt!drGZS2U`?8|K4?x0xpr}wpC7?6%#!^K8b~C8-Fm&07}=qyy^CCbHvE1 zC-Ni%@9ER0)<5g!0BbKUP61Fq01XF#DZu$!#tB2}arB(j)V#dB%r)XQw>oa&uQ|iHJ}evwoE&Y>qKI zG$p1AJ+I*dQn-@g73F-hp1L|ACDGY{H+!MhXkg2$*z?tDea4azT{DJ}~nNW7eHT@6DuZM^B zK%|?Vad}r|koDzD77%9w1G~+)V_&x;o?lI!{8 zk!xC_(xJ$>mrIC`09v@SBRG^fY(6TfsCT=$%#i-@w}9idVjbAQ=wC3Sgs*LWZ&O?U zci6~F_WR)CWF!f_4{M8>K)O^i)VS64kRN%?x6qjVKB z#DzUdt~%9xVr}MWzs#fS*RmMmKE>uWH}o#K{~s_d4{B%Jf5D{CsXRc!9Mswe_w{%# z$~S`m$wU=Upv{9oz}qL<5)7l>DaoVjxbrzXgJX@7>R#l&O@WZA{D)(90O%40|FJ1RcmipwYj+_FgGg0PP3DC| z2aq{y0Dv!eeUONVX!0GK3GO41*VQ7De;AumH7d4wuy>})b@Zs__R2zrnnwA*#cJp~ z?uaCXX=-THn%y*fDgXoUE>NTa2Ck6I*L1U+WTQK) z_@hC2B&9R+iJyS$!Qi~7`gO%=r?DJ>|Bd8m{~M}9hwN0o`?|N)>$MquN9e7^Z0vF; z^oYpoJ7#-Nj~l#2c(U+{o%%{Q^2ftpVTPk{TFCr?I5h*qWIYrN5X6Av8HU|MNq}Ok z4-gU%g(4`mkVSzi**}EbgZO`o`NI7OZ~3|f003dQAu-|7x(5OyVTrqZ`Ox)zLFAI` z5{db)!jZVrV{gY?>Bt5v`dRjxUdI|GRR*CFgb;US-LH2w?2J-nT?-IX40pBs8pZc@ zF`gWL5Yq!n;S@o4`d1-)mK49P|N0Bc{}MtGe|G=B(MbTD2^@FV$bje;7&sdMx)^+Vc|f7PRVfUwbEx+9^^HpZqy9!gA)a^&cCh6mbHcbT~lSQhD!Qr(Uq9~ZB&p355z0JjcoX*J3y{NqiSqy01iY+d~USD!p*p%{|tS4 zw5Iyms%4>X@QfIr^rZ@$CV6>J7ewoYWavLAoRM_KY@!%Y#7=!2UdjT% zTA-4@@BmeUZm*F5YT)GHtb0AGNn^MyRPPKV8E(;6f5-SN@=0a_5@6=(6;h}_cCRDC zC_uDH3#}Ip_fRqS<}*x<#6JSyt1{+q;{(2Aq-Chx-?DxLh?We}DOoiS+I*2`U*P%q zg^!yE#L;T)gudZY{}s)l5Yx{U@B_uL$4;02CD$toC{F7#B8%i7U0zp1N8)K8_# zengh~%#4+OTIsJxzBL?_2^C@>O?rK$!xdK3fUUo&A*Un2^WisuhVrZ})1JGtp&omE z&QqFYaDS_$qcOO`o0NJ+a^{`c3#RMn@Y%7?C;{6$+fnew*ah8fXy}`%qe*Q5~;DN)Wx$nl2}EyYully`e4vC!LmocQUW-omOuZiNyngS<7zo zf((kure_99yqy}!yH8eDd~Rg)NN^KXG+95&^`Tg3?amvEfc>*@OWjkN4D{q^3V*Ie ztG()RWosy_Oc!0_y&+9cy{P17SF;hKd#){~VICXnuBp@=%iDUx$ebOqnfn}Ozx`@x zq#`uPUea!J`bU)fu$?xAVmLIWY=k2=%$K?USWRstNVYg9)5iPwRda)n8}Df^6^|y# z*kLiqy8g08sW--dp!t>;lavdyKN}Fk#e+~O9hQTA8|mj8a+ukgTPq{=x7E)^%1t5S z=p>rd#+lG?c_|Cc5M0^()ce@V~d`Cz|Og3Qv%;n|_`#4Nv`t?ukT(d3R5)?phtd4=MQ6{8c{Ya^S+UM+B&`ECb3 zQ`8d#4`=%Ph-DT>mTzTbO4#6+#8$n-=8AGMD|1N}Xjy5=_{5IGjU*jZ3JsU@)6e#+ z+WOHm1#&`YY^g6C?N1D~~I-Qp&{QS{6Ok)2hB1f6w3T2*eMp*u?`I`2NZJ>M8C2U^QsX;qYk@Fc>-qGF<^EQN+Q z{=~dqGsHFI90qgr`t2DTyzDO(^2ZUC7Is)pZk+3QFdwa2yFp_#`V|nmq0x;imdW)h z2g|y}KOKu*nEeL2o=LC@480|GnR)QrB8R!Jx%{hwMu;Xpi$vLf>{eH*||uY7a2!3YW&S4Djn_oDauHjHIXL5B{nPRKO*uPqvjLCY(XoSCR! z(#^@rUH5&T2cnuqP*8hxG#(?3C&WrXdDI}}ZQ1pTyvW!3uPGH6e^xX#C%gGUUY_Q* z#98_v!%d`~E}}TsNbI3pw3ousNP#}JG8=FOI?lo^m2}&_F{E>+`OF*?hkstq<+fH zi#Lx%#LBGWsRY$TvmIdotf({^V8NO5c$_GAu&A`$`3HBbc=z|jl-BvssOnzyK*Fzr z>YB0P^1ZA}>VDbAxS~Ec-OpFl=Lh@7!D?yqCmfvAkUp$Hw}i8_%(5+Ow~4J>1_+ML zsABY6rGBP@e6O#!2w;E`+-$YqJ`Rq81u2?wxTSfvtjvCKyhbsYo-e2_v<+nKj8{iz zuMFAXfJ4M2Sl}@d8A3wH^Saq3pHF*5=;@|Wv%#`+pO^m4irvhDqP)T%7cRz@?TYNxqpCEJS+`h?b#kmiZf;G+=SCi!*&S}n6Zx%dh7N}h zdJ(Ra`5!&-D<|;KUw@yI9_)V$?7Mt(KkhK>$v2L59EIe4LOUjOD!OPjNO*dWtf;h% zT69^%tCQlqgcq@$?lET~;UED)D2_q?Oq%)nq2b3;(s;WcHN5fc4Z6>}gnN5CWStU`bK z&pgA|UiN?0)K*@-g549@R>);Z8`F+DmdcNx1i2cTh6}8 zS)+sud0UIM)jI1L5gJ%w5>EQ{z9*%Hq!!W}^mVy}g@$USZ+qo?R0Sy8FRbpT3YnGj zvGX6Wz7iDXwD4L30lu53xF852`9 z7_8#(w+xLKKOQBUwa`u8W|-C@0;?-vhXwWZUA${qMO>(mTY2s7zK}ks`WA;XGC2Fu z&{2CA76&-gIfRd_Ui|jDCQ*cROMwi6)5)Xx$Uf=V`&N?01m*7Vv@5Uj$bPo5Yr@l= zt!2W<eB zm>va4P_SgsFJ;cc!!s+TCXJS1V)9UcG~pME9d9hgX<u)`_kz)gI`e zQZ>b5rioG&E}a%A|L*3LFCoiCycyrmQDt|KQMUm`OsYXEXGxxbWFrn=k&o}NE zi*M+sG(xLN->RqriDhOuW(KeIu5)tJ}?8N4w||`gDVur{@8frEz^+_&j+dL3W9ms7o6a8A;QhBVKooGoPPrMIGY6^;mQn=s)|}4Yz#K@4IsN4?Zw&eICqUz^p7-)ujn3owJAxCwmvo^x%I(Y0CTCNpvE}uB;TSNunGIGGn<7L*yY)u2%ofX=juz&9 zz9aUaI$X;Y#BThfGF+BI5wHjI^~l&oV*Xxjf!Y9jr;aa~QBZv`$4jPI+(SWJlq8pg%xF6Kr%}}Dw@m%=6}Ub4G-kq z|6;llHGWd4ZTBCSV{jdI269(>+frmB6$02q{5`{8HeqPRoU-Z}&^=&ji~D;iP|R${ zDlroT9=->#>^4BQ7}j}v^Q;4`{V97~2`uQ*_Z{(Be={QATHjOqW4v;Q}DLzv$<@Hy{1 zAtT|j9eTeQ{qI|V@(e7E{*Be2e5YqhP>HrPQ@R@SRY>E7p{HjhD0Lkd^Yi7Kn8u-$sH13JOMbzUd4-_JrxO*Rqg zZ&3#NLQ+8~-0C-gfD1NLjVJ0O)|Nn>yOuTZ!UA`y+E69Bv>E;hkeZ?CI@{WAEG)b? z9QA^y!0DP`F*9LHZGVSu#dT&L1}J^aBN@1zo}irMFR0JfP~bQoN!r`n10X4&G>Bo= z0vW92E=xfH8j$?m*j?P)G=)xNJ2RF=H1<=SRPoBK( z{ohIKv3VL#g4&w84^s`s|3%vkj)A2quifF8C74yXhO&Jbk*P$5N2Nf_xO5EpM#}4B zJ&V)OKuK_M(Bj*4eVJEKAZME(cEu4SUXyY7vEl!(1!;^Pfb&&y~PgZ2$;0F?D^`x>z#C&E4006q)A5Tts7GgYgQfVl++5LMRpEbn@^;2Q- z^Vd2xTRA%0rRiFjTVj)NjCD+$A~iU_G$+ag#n)aeq_ueHNYc?xdFAEtem^BD%82E|ZeS!m5|eCYd30ZL!6_!*wpP2P5pSYMM@8^6p!$XB zril4rZjaTBMk9#Ie& zxhEx|+vvehPoKbc%wX3c8_r|bBTYMopDlmuyc4_p8*t{KH<^UZT;MfYyzb-RNZ-sY z(yXM0(9+Qt>(&}vR&xR}W4FD&c>K;VtJ$*3#-iQk`qK}l#?e4u3_v&@S8xfsBk6j< z$ACVqrDYKS*C0B21o)nUW&c>$J)_0XWthvWbJgh$9^*mAFg*rafD{54Q>|(M_~omw zk)`zpNR3T+KcB;1Ymiz2L3EU})r`Pp@M0p*6?NS5Gsu70Kf&zspEo^VqMc0iCsMVv z06b10Ob_+peL(U-OiZj|11R66B*hCgOFedS*#OYn#>Auv5aBE{+>-F#FaaH84|fXk zhN6_pgg&<|K7anai;1h{KD9itIlzQN!aJb`a&od1?HxFx6rg|?jEsTM;K`AR?D^gQ zih{iutt5!pD&X3fQ8IDk`0a2*mV00a>3zb?Y!j277caoY@Z3^eup*fHA~C+2oYPs` z&JF=IlF-n!Zog?;M>t!(cEKXh8(T?LOS-FkSA2 zURTjpxfXj}T?#7kEVTj>bCAcm?dh?esp2j7Rq$7Mf850h?~+t zCIg`Nyal9BKo9|F=;l;j@6G`o{7$IlidJ};dFRh5M*s(PQj>OdtsSzDA87_tZ2nu9 zMP?3b?Z_nWH(go4o8C`{u{1LL$##u{XBJ_X(p&ondHXl~?rV+!#qjf_*$Q4;&Jzx-Vc_4eR+N9f~5iG*Bie~RG#UUE3;=g-_JWJ=1)coc+$ z6ok&l2Wd{EJYKw(r~Ra#eO&%c{I9dJVx*-plUq=Lk%A7S0)}a2z!rg&ae+xT)##4R z?|Ok$QbNVvfrfcaox5=5!pPeh5Gmh*@-ZN$Dn{HslIN5>TGCn}%r_$=y3z^Y(knZP z#ZWDTP9jQSK#vo?sN>1;eM_sGMuZe5reJ!aqT(MUH!tG4fS-etQ(IemrW$q;&B#S7 zuBhjI(W5rO^NXGX78i)hglszV)q29{dyV;J9DvyMs@5U<&DD#S&%T+m4FX+gAh)Hc zvYPjw9q`sx zWep7t6_xVp*vS=G4z6efys7A`avw?fi_5T{=R7htNLTBuN@|QU~5nLEsjH`9qzNO zvuVBpeoVBhk_qfJ*Q8bgI=DQUh7Zm;EMRY~#d+2%RZM zR{Nr!o*+&sUKz#tG_DW7cWBsxB=J z1R6pC(>b{OBNz`M5z2-s3gfGc0*qPIO0@{D^(=ZVOXK?*ZR?@`kG;1Ht8(qz2Qd&7 zK@gGdk_PD(q*GczK%_$ukY<4*(%sVC-HjmK(j|-T?pmz5Pcg@>0F@35vwYYDV9`d0GgD>d#T=~OJ!<>_1omH%=>W1me1>3v;JuwSnFqo7LQn=; zT4&Pl{tDiWg5_T?1HG${x&I#{s5_e3ya28Sbj(0^=dm`o!Sef94vcGE7^4(-vs4$sjeQbU}N`@g8M; zeejc1FtT#$edBlJzJmL;e}gI~Nu&|~6I9vV{PKZy@A2jY@H=EA_>KUVo`Ui<|W=^5yMgaiw@9y@eK%iRTTJ z9AUR53khRivGTEhU@J0T&DdIxt*~HaO}(I-uR}zE-MOC@2^iz?oNmb_C0(Cwj41Iv z?-TC?+#PvxOFsz%s37>o@{;N|)m2;dC+s`$nJm803FZ<)Vf!hF(3b@wR)B?*kLAnG zP7U^-fz;g(@V^>|`WJ|qSm`IB86+Vm-eSWmO+g`K@fM{A`cuQrDj-i# zR8%|M-PorkA>~qUeO59_BmXCp9~Ctqdnn^%bwcU65-StuiQR7gLVKX9it0V;8TbmR za7WJ?R0hD;8?Rhlh?oFE{J9&^;u!nKa}dh3gI4;^wib3@V(KV3Uge|N(!IRA0Lxj< z&Q4#yWi5;v1P9sn{LUxpM?;guQ4HlJ6-4Y#_vhZFkdYm4O)yA2L$+F zwlrMaH6Xtb#TZw}0x|nHEV7Sx{)<$!yQE%0iRhQ?=D!F@8IgY=%UQSG8y@1=3pG0W zJ#jx>eIVu&UflG*0FIX)Q!x~Lc^Mh9XPj;+4jS30v6XpR^-_RQHW;1_77dD4XSDzVrdn~ z0A6a^Z<-q!h~{&$Jq4GS;y6_yn6IJkoLiXb#@psbU(RQ7sRpz|(i)uA$a@cW4E>LZ zF94yU;1VKPWDTM!#sZ7s5)kL$z>5i*CmK)e=TN~azZOx@3j0e$0rC6L&{PS;iv*yk zmi*OA1N;N*cP@|MbPNn`KaOevl=7$qfVM%uUNJQ6+qS3g@V)YlU{sFU*GhNU#w};e z{RS5C<{SuEtix++cu4j8rit0u*+bf@62( z9hV2X8s`b|V@8egH`OBU`UHn;Y;1Oy6`JDWUg!Y3R$|;)0;o$N?nV?L1rkc40G-SM zn2FxXw>eeMF|z0^pKGmIN8vc*Jv@8_|c zLd^Fl50Hqj!B?ZB9HBAGub=lU#1rYUb2Mk?V|dE$ZM>NDisd?@SHxm>_=-U-+}1yJ z4P5_6C5W7myc;9q9Z;t-b?cgL(3)0oq zRbN2Turcg&0Q~QAKDX@?P{52eY82~6Sng?Sej z2q-L2Qc_s}sQ`Y4-o6>v&V8(UK4<%P?;Q3nt9Pl^P1m)xXTU6egpZk@le50TnRIM9 zXV{Z)(27l7Y59Wy2c6_)Hjo(D+yv;rI*ZJs?0TV+r z*^cm`a0Vzepud4O=>mkWt(@f3ZTI4btM*MLXt6dJ6z*5AA2s^VOH0YTs<&=Ev^2U; z#B&Hp6x4x)6IBLg&YDw`G@2Hy-=ji$zjqnFIQm!NiFu?>uGIhnelZ|-^OVe`zW@p< za4xVrE}S-zS>+<~WLhB^zJGF(ujkqMKpghXLL`|uzSgmR%blG>Cj~+1l}GSbGx*Nn zW`BYrnW37c2dT7pCXf~A?(9t9a_lmO7X!f@r?6-dWB@A!Ni^mf@y;8x*Z`ZKdr=Qd zm^Y;AdupK(=yxfJhM(UdX#2Svae!GTvf0o4Xb*pKqV*`s)(oJw);89dG-`TA`}%SV z`>%1TL&ItThoY|uO0F^XZHs>@iZERi(U@BsPcm7|C-P6c7mcyxzsW|e!=WO=3NZ7z zhqbQ{j+8D7X1D~PnzK%hC;3m9bjsgM0LCa_KvIL6$tVLzIQtk29zvrc%4Go&7UTfR z%F4*90zpCK9^(^JVQz_Qv$NwN8s7l?x8CJ@ek=;9{>E|uAUgnrd=N1|PnDY>XNtgR8vl?la-T{EC4ZhheYGMtLZf*YO_VR zt;rDpZU?AdrO+_@?Qt<=kh#dz14MSM{@i6&2PmeIF-BeR%#EC}8a;Z1#V`elK?E3E zn|GV1`%8X+G;#fzPnbZNjvunV2oOU)Px4nCku zlARknb#Th1$y>`QDG4Tn)A_kdL7`ji%>Kf{K?KW3uS0%1> zIT~HJa&|TURq%q{TEgK-u_!J)x?^Pz;F=N5`2Iugq7f{xksPe@*MEsbj0FZj=!Vt3 z%$c{{4Je1bV9UJ~$h|`31VO`WHa|;7>RVG+<*SBo=*eN8&rMx9 zewQp=UQ^*W6{*oxuL{8Q^!mnaz1mOibw%OvqOt0+?HyH_e;U9-qsYGokdq{hNAg+~ z!%Is`E1zk2Aly?$PtSEDZ&yIFZ!!JSdUp2hQ<=rSI+DoQD8`o}H$N<%Rxtj7_TeH^x=_Th%I$L- zuPl$$03V;euCA1t%0JaJ^4*3ErQ+-!>9}4w19<$LeB>QZPoR^yb3jp!!qLJ%gD>(T zAi&Br?5c<+fG2!&j!KTdw2;i|4uKV!`R>YF1O+KC|vVyBl(F!7=ON*~K`_)FoAvr;3|#v?uf zi(wpiU_1}x@^o%L-4Do|l~?WnS{^Pjke})NyAsFX)U^^vVTil|){lI9P}O%8q}|_3 z9cPr&0)?K-^(DXQ=~r{N0jq3IR~P&NOcaeu-39wnL+Is>*4_WfW02>*kHc0~R`z~~ zX=5RQpMjQf`M?3BmDefU>nZ=k_rl(Ou~R@W$W)EJTQw>B`Mn@DS!9J5%6YKI-CE45 zH+vb!&1WwCzKTJ_F{S6DLS*mH-=@LuRVHprE$?JObK>Tn0Kj=rR&k$7n3|lpyYdWn zsiGz?U(}XSFjW|w!OPCP+<)CI|Js0jxPs=ywt@(XvqmjH0A847?HGc7i{fNzxHq>s$6T~zOmEQyvKKyte+W>V zXj6#a;Y@`TQx7D;>v}_#o$*@x`n2Cs2LgpG!20ASb6qZ!PlarRc=NQs1*;>wlv-9T z4L(Xv($^UUh%_Q~4P93oMhDav0LyP^SXEV3VX1QvW??MhozQT#ACa5))*PI0AXu&C z;%3mUsk3Z!eK=hU*qk43+JR?;>sz&C?_gv07lsBWG5;jqP4vh!fx`AbkXq@ zA1TE@7yHt58$}DVvQ7tiFPa#bcKOeAydJHSO8(R!k$>Fw^RRx<3v>IA~q z7x(l$(9yhr&Q*aGGzKVijZ{sznw$54t`j1YPNQ5E_VJoi$3F{+SYB-$zvXqE+S*!J z*jgZeS#5Vj+gyOPx)B@-q=pg_X3Ap^Cj8u#%x*vMHk?N6mddikHs`Qh7ESWW@jMTt zibP7~GSK(`NOhxIXQoSrb;N7W*kB@X@Eg~vR*i%1Zg4UXCgFE-R2230N5|kxh{)k`nY)sO{tyoS_wgsf&t!Wrs z9E*s@At^0Qge&ZhnnEh!4k{5xtP7YNhQB9=3GJ8Om1i8rc1t9BOFN|*VET$ zLAke?pmmb>>M@%QpSO{*^*IXlkRYxTR9f zpku|LSts$1Pp9VHToCL6RPn1{Sx8CkPmbii(R*WRo^W}*<-ay$F@x-q7^SAJezu+H zb~WA@)hitcxOdxYVeLSnXU)V!uuwhz^xQW7 z!Zb(emfn^BwZ>S$J3I2S6~vEO-pIldfID+%r**Q;6Ww~j2|=&x+?(5UB-4u4@uGqO z|5q>{x_AmC;_$9>Ygg3x>W>%)tKX3$1FTwP;X%ViREQcm~)ARMjT zYN;t?YWchlatnglgWmcjfG3pkdSl4+^TxD)n~C;TIs z3)+ATLMSDm*mRfL|DxD%o=U43QBw)){v4nK*L5gP;5c65YQjH>uPJ61UMf5kBC( zv-}H#R097~j^$M5e?h7EPnDK&MMXs($K}mo6DXT4$?r~t?Kd{@q0*gM^q)AS*$gy~ zCP2jmC5)B6xW61N);&$oUW=-$JNArQflAmTEG%~0KRgJ)CmS5FwM3lglOY zDZG5yApoEhDe!SCfrbV#Pcg7RV9cF>uD#BJlNZd52w;8Vu6n_Z9B%Biuk5*7%y1p9 z2nht)U*mUJ4<i)aRhFuAwDkt&dgcv>m?BFyn6K`%Rq^>6i7C3dz8BE?9gNW zYk87S=-HXmc_rw-D0VNjxnTk9(|WCazzZ>932f?2(r=zNT%a4A8+-8p1v_7(I1ase zW$)�@*XCn23m@SWZdhH{^PHKsV8-KkhN3dtBTrB&2pg@bkuH{xt|6lsh-skP_v= zMs?NTKq|1UE((AH2DPo-k*Gv`i{o?8hnH|b(%aX^NJG=PF`6eSFF!Oiv@?_Hl@c0C zg4A{!ph~!FtN0u*PruIagVzAnOTbt?#^>s2&x8hi-Dqyg%FT9mbS%`lP%@V(1N_xO zD8jnC9}Q*$na8;OyYX_PonwYrz~>Az$EEpK7)v<%LkIrXpm77#F`V6?vXt8^Q;Fbr zTY@30_&7kWEh972@6m4}+B5R731=HnLuORx5~O{al9lD%z0@G*V0+brizq7EZKtYy z)55^OQB<^RY7rpJzzu?K1jP0(UU0ra501VyG&>&v2e>mVJdS2^TMp8An5`r=wHMNnxJ}8#J zuOQ0$B07n#AYLBE1LCq^4AL^LR-`b2f`qUHX)W~o@Abxp^`$=M6Z$-0TDY`G(3zP5 zv6yvge2-ud!7BW_D5ZNZE@pJh7l7k@RqdME=4yNN-Yky>aP9q(qg?KO>5}-H6eThx z@;{R8bBeJ6{T&%&@ml8z{0#7QZjKPZTrLXLYYY6s!hne2?ukv40g6ZPKWC*M-ckAA zU8g8~VYB~)npd-Ka5z8~_b%y@!IHayK#=$quP(+f^~H$pN;h~)bF;Jk&gi&8Q+tpH zIw0wFM*SvqJ+sz7HVwS-=?-|17Uu?#Y2Y8y+6w9uy93Dp2ALO<2O3;_^z?l6n?UYz zgTwsNVstZ>^QJmg$DbGtKSE<=)sZ^#Y-XZDKqpGddXoNhFx5hYM|VDrqoyVSQU~g@ zO-t5hX+T7ahJgV*W-NB$UpM}tjiOY|23`0c6#aHIxI_4f_R$02r=N26Ac8gXXff z?pDoHR#K|PdZf&I14yD05gQpnlW%p{o=yS{Ia|q&kOn$cE`s+rXr8^j}ZwAo5OuwsJJ}F9RpO7zyE)L;B`!@!NJ-Jul4XM7zrkAX2I3A z0B&gi*6i#+$KBsAi@E#j07gL7A8LRB4ebY2PPYBqP-IdthGt{%H+n-?wQECw-fbE8 zhZRRJQUR;sE**tzF)b+s|D{r)Dx zznhBH|9=c@_&<(weqYr8E132tb$jyq<}VA>wncwMK_3F$%k0rv3}1_DZ5{Z)W$LjQ zw;=2v{621Pg}_!k1h~%0viDB54NM&zjiHkpl`U=X%UP3!_TOuLAmv(lO{U%!ygdl( z6||zw*V;OQFG1O!no{9QHJ-B^2Y^BU`uX~-W{BVuz3g35{FDU_$M*-!ELCAKG$TLC zUtIq}L=XTue!oX=Y#S%I`11jyre-I(<9C_`4PL(I-7QIseDl;rlbD<_BSX$g`> z^r*zUES-Mb_|2XX&D@S|O*0c0XsfG`8q)FIMeexLPv?2>H_TJh0mCM9Ov*qCX@lta&fH1!a<`+Dtxy-kum z?u?;W{L%Y_m&cWRO6bCvACbpjo>(E zt17M^E$AW)7zwGzH3Lj69p^{zg`U2}N@vQOlB>re3XMt6$YP%9%ZX_Ra^$?W5zIf_ z-_JrTkx73999jhK=EDlyoTUM8N&Y|pP~fC!pIDEM{n#0TL`PKTCU1Ys;}a?YB#j{p z(c2kmnkLzaxZ_j(%_6c2#Z@JW!LI@50etmm8>08zr`k>&K)C|5$UnH3@%QwK5v0-X zW$73M%ne#H{#f*fhw^vMPE-lQeNDC1RgU~Fg8+e#Hpj6Ou@w+=C-+VfFK&~+os3jb z38;5Hk+g7iw6)c+>O4t)0JwgT-T;qp%qnK}tJ;TWEpGeWbTe2H=H9Ny8A%dkh=>11 z;)CCQ_mAv6vMujoR=t@VU&7DLBjo9=? zkE0$6YtUU~PU2b$b-naBfB2t|4=5Zs#Ng%7(4v(wGsJOgJJeS@ZP<%a4(|i{&_@~* zhzu&BZFwfw_~z=?Ib`)ySmDgQhpbdtVJQUSrv|20J|0~ykKv5~#*?4(kY36DY7zZA zaMl=px^MPFvuw}o_jdnm9{3-SUS2%X1uO7`UmCrReoGj?i?Po=a}k!nB zzX}Mq{q>j?t2vRpfKw7G8wY? zlGiGxKYyON1Ae<#m@9tjX#SOygC^x+S>U4kJmA6e2iynypn&Q-ZV!hqxC$p7kGL*) z($Aq#C1xtF^QR_R5ob?gD^krT?`8UQ^%xMZ*EY{rB8qBfc>07jCXK!%;*sx#^lX|} z78`L2b!kDS?T)O-JqVlgz-qy{t%S`ytdP^e{dG--OkPDOPEjhmg{%@^J1#AV!G8oPE-$ca^A6ZDT}_? zOtPh<0;3$Wxfcqr1}btZ3C396H2Fuw)F~zO!}bTTt#t&*=ke4C}xz$Qw7UlZ7(Eog#hW%Qiecv z$M?`fRS;FoWxyw%&UaUVq)BCxGnoSSag`Bp6j|NfclILp}cm~{-3GrI1-A{?7 zIU9kCWCE1uyWV`4orP<*{La0Zr}q0qz|Hkt-*9v9TPUR`oZr>nW0vW1hPBeMe_D)z z_?P>eMx@${_jm%`75&Djw`?pVo2HOHhP02VJih8TO-{sq_G^cV$oe z2e&hPQA{AQoyv7cuAT=i+WhrUd4#8&kYShKRAz!U?~_ZIYm;XPYhBUqbx6U1sIS!j2v}&Y`+trnMdU@=F?|F>`EQvPI zSLy_QsoM)t>NUrf>B&lQQD^5ft*fk4Ik^H_F#R8TMbMN_z!w$_&?euZCfXc@IP|_? zT}|EUt!|vvgG@o3iN$^@r1P0RiOYmG2fnAY3SlQ_@ZRdQy zhj-_#mKgl?OnRlSSgdB;!BqMBx=p|Kl54ZLHCshK@ok1_B2`QzqCVTe;l@%0e zq`LwkDDUB>)kBS(@>?AH7r@r8ld1M7;g`+Q}tlZ}rB>b@2%`aR|?+;@|K%GzGZ;q?!(;7A{HPWVU{gFvcyYkQb zCij+`j_z!xG_iOO-|NMdrQ{UG1%AR}dG!NrDYuRk|8C3b5jJr`XtpHatpQACQw`qO z>aL-4VA|0iSoXD8BDO7r^fI3JL(R4qWf{zHHMvm}CAs3f$p!B|HPN#;X=RE6UW$rL zwmp-zfm+dt=d?8YF}n?T9ApZ3)2QRGCsd3BvAbr8v(~-+=#$g8c(})~#W)Q~4FSp3Oa$1BMG3`kJ$jPP(*~87hyutMD3;kDOi32 zD5tCoOP_Ll_c`8fxh!8U&+^XAtuYKI%JV+>Qjn9Lc)8nFFQ}#({&>d`O>bkMqw;u<$0nN}@V&O3 z)E@p)K|9)V%1F7e(9KYZ?M9nZM$ktH9UBAqOSV2BXT0pJs249UUJbtBU|c*oBW9h! z8|ONTt?kaac>P>+CC>v#2zZLjfEziHH!rG5$nKxk0uMsH=B&QHpRc$zS`gOMaVM5w z{P6hP8+y;boqU1uQMhtX&0(io)H}VYL*OV`&e8UmvB5oe<_W~mT-74~jA!#r&*%8h z6Fb9`IKYJWg}MS&oTs748zoV;A0K+>WPqGM=+im;j1thp9_{odj}_?Q+th-Y{ZQDP zU)+$g#hVR)$%@|}51NPNaLIe(&|r91%rrs0+)CPO>)Y1r6(HI;4u{DHN)W z=)E^F?@7e8`olHkd{7%Iw&^;T<&`=?FM?fbVLhj7deNaNCU2}5lydpQ;J3dViWJ3N za3u7vWUpNb-jKyA?nw+O)BNapn*a5Bhj#B@`Mj+;xKi7j3J!+`ms3)Py&qu%^L764 zmDk2X$Y(27z~I}248ur-OyC&y>G0pU7r-KAL%=@r%k)DGn)C!Js115`E0;7IrNby zjwvuq>5a=mqzF&~etBHDF{*Vlrmc@1Lv2t2fYLh*q08ZSj;3 z8k1KgefwHNTAP$@6D?Q_( z2y{h?_vr|L&)2pFmX7H!yI;(P@DCZjFPHnhW$C;N`|KVyV&MddN7l|pK5nQc-d%S6 z`l+$vzP8o0UYAA_TYG!nqo3#0gMN$D&a@`-ho2kp(OJ-p$f^r|PVIr$*Os{8k}G^{ zm%F_{pyxa`7*tKxYlV*1KA7$%YSomWkWnj+qX-Y|e@E@{(%0HKa6bj4pS+bHhYv73 zC_QylhuF6T-3qm91AS?8Xz_S52%XPe;Ia8D5*YLP&AA1%uwW&1)x2SiA6<}OKlsf# zd!@wb2sPzxUNbMSRhGmUMkPb<5eP-f>rjWeChCT$$d z1kWvC=pH?mAzW`JJ*Yv2y3>#@`{|(a)1xGV!Y@N0A#OwQe4<0^}@yh2O>Vv%A;1JvQ?Gm#qaEc7{ z&DTJ2omblzU_BUl{^?i?vH~m6v&c1_M-1ahOf#=_nq@hIhA6B<-jK&V={9k-;Sc31VWe)?iMALsTg z<0X=c(`W8%#4nwO@%))Yu44JzT-nbZbp=QlMRz50w?CQlc}_Xp$DW~BSh^J=oW8g0 z`l0yKMV9Q1MD2q!E6zYW0%rB#S@o}pU>;W35RdmkNFrvpzTt08-zjp8LLYSOIMh}e zyQl5cKfTOvgHK9vwal{K=kA}NCAhkJ5m6m>;=!-rfi933h2vTcDa7a*>xE;>@(+bc z(4B%|1YXHa((7t+Oz!fZF9i}IK+y!d622c-;o37wvc8l|6E7vWj9$EZy*eFsW>6CjT*i+q8f#B(O6#oAGDeya^Hr z2izLVPV5c1#m33cJ?@A<@2{->8;8K7x`K{@9;n4l3>x1M9R z(I!r$b8?j!1&*_&&u1WysI>2uMdoKx%wkKpL*UPMY$If7ZEGM3P7(0t;nOff;5qC^ z=UMMeps1pt8k%T(aYFp`^YW)vXoR=#yt^M=tG*B2|8C*SSbd| zZu~=L9;&aIuG+q5j$LIv(PjVCmeA7m(v4D0&6+8n=!kFCVXd7&+txr=&a~$&UTDf_ zSmO9{v~#s|iq03MD*f|a`+glQUb#~g4%EmqPE6a1)Zg283N!W5J|4XCq%g{*qX3-t zAX~6?fplyWd$2mf`@30DTy#9+Xgd3I3Km@@q_1zw`h76YkpiX-r}65XjE2U~$BFQz z1vsr6uj3@)?!k0}Z2OZh+SOIme7S~h^hSeR)zuG8TT_sfa#>ZW{if`lA3pu9!6Ru<=X zYxCJB$qm56RGgk4HI~>q$Sq2mt}_=HFX6|Boq|Y`{M{ zI$sFvRRKlh#=ysXitj@4@ln6?=y4>@htD>RAWxsHMtv<~qQAp&p{VNRx`1DV9r>;Z z!DV?o$V3q@1m~9VzOGIiTk{uDMUYU(L1?6>T4D^GOzg32sCU5I^PKNTI`pxL_}eZ`++??&iDEPOPLJic&|)XbCjq8`AstkY8Q?{CJMve#LUAGC>^~TGnrKu z?f@59%9w_-lKS=ys1&F+61y4^r`4tGgO?WQVLuc;XBrpO-GQ7R#@~^PhtTBqDM|!^ zM1NCIm|fB%An!|5@o^JbU-7e-1Zd`ghzhklY|#=(=3$G_OY%pmD|<5~z5~{JuXg}9 z6Fl7DJBVur`Dr=ZPtTF0IRzP>cNgGWo*m7;C#`^A_ge z_Qn!rMq!L~r2Jjh6Gr2y5luPm5H&5D>{H4>a%!l-sy&DehZe|Vc_M4rNty>Be!x|n z%A=iC#t(0(|um`!=a^Vvie9r zqpk|FzE=LpMcek9dooSxL6)PIrz;&bQ{qEs-phd;5Fof$isjYn&kw|FpiPv{w&sVq zSN6o$c@^;3>UFa@iegn-`Dtw6WVDJ-&#sL?2dETx2(u1?#Ea$fCB$ z<>hiWlJ&l?Qc%lMgQJ~eZ#3NU+nkI5y5^<8!lfdHtGyQVyZ&hU16c%-hg-(~t?)qV z@hv2_Xy-;{+U%hV@m#)`MOBJ{x$WWx0;*yOtgBmioPqv-uGPob$I!AMRLoS3JRjsY0cpUXhB z!DG19&v>W@uYG_i4umzrkiDKGn@o*NnYBfJKn*iX!@XR%>t`Xrvf>67Kg&}s=!%F0nE{rS)Tk&+TEbn0H4)16e1ijpR zgWxaV2EWn`tmHH~%LCB?|7K&G0_=@sP)K3*4@(Y5`d|M8CA};4T;JdJ_b^d}$ z(`23JRBoVj-9>keq-hzpvcqn_pAlK^pAYtY3Ie&7kVFw$cP>0q3BZ~%modXQqNn08 zc%-z_1bvcg@=%zsabiH~xwxpTw5V+NRySp(T~AI($r1-91eelw|Dn-~l%U|f&AgNX zSSyoTuIU=4@FNvv2%vKXHqa>U`hyu^@{0-#Tn&#!vmkHIgA-ax{DDFGfF2*O^yMKc z0tpS+gE}DiYAOnb>*Gpff>?9!7Ke{|U%{}!B`j0$V_lw2ro;xt)d@>&f_Y~hcXU3-6&z?SjcR}(+S1~J@n}se zUDecX<_h4tD^9XF9urza+dw=B*K5}7lHm9Sy$$rwVnL_PPpt= z<0GF(x}R9?Oc=EpE{(<%eq(HVHLe~FlgkgV;yt+$fn_kf>>{j~I0JdSYt+__q=L2d zc5?Ep=d8EMz8TaU)-NvJe(N&cUc@TXGnDFKVAs-16*&RHr$ z8ebLEMnl99VLWt{JQ714*1P+9pfCmUQLAp>2@%S{ra|TQn->K$%-|Y$4llZLE@n^h z#k1JK4W8Dm-{}b$bp&%pHO?-r%yz$_daKs$%_mSThb(VDHHY>N_U>%KHavo(D&f|I zwF7--GN*GR%5QiJzk_hNDG^QP{8q(ts1=wsX|19nM6fkj?QInXEo}Jb>nRF#UhJOA zyi0mdiq=r;e6dlws`#ywkYR}iz8(s;j-x205E>aOvhAB5fWpQ-FrrT_;t4vTVsaN5 z2$GnOQgEs6v>M}-TVt_G*Ms{6G3^8h)atFSHy-BmjLA({$NDt*o6BkY8$Bjl?)=OF z_86NL(jk@d!!Y8Rs*{y2-$7yLonoS}F}9=>V~I4nL1m2J&|`W(l0F|FZ+k-w%!LVc z_e@I!gu9`mVX9sCpwJoq{=O-AE|k2?pMLY)f0V7v_?bY#6$)r)#MrXv%{L&~Y3rT3 z%zPS(McmP-Ia7&@GUN*aj4n5^c>%uLYG8|@7X94C(N9)9OSF7%z$02Hhw{ZNAV8WPZj^Rej%D1u*m zpo$UegES4S)QLjk>w$)A39E=sfTJ{sar^PKb;^IKPSt0H3`FgT?q-kaVppe&g6(p9 zQ%eMyWn;5b;J{FF6kEwd^)l8?^2~>r*Uz-dtIV)1C##KP1$ja!!l8 z%sevywf4b8afpQ5H!7o8?QglXT?LlW;Rnm=Ahl4iIFIK?0hhI`RhM;pBw`wrv`==l zml~zB3Kl`8$XD+)`@QuCSt}Ej2~wx)!JQxzhLnWXmX~6C>HM|r3=C4vkHvo0_A1Mm z50;**$kyX^YP<)`Uj;>KCaXJ?=YRq!9Bsd?rKN7=JXY5nHw+RLYPI7==ZCcabDsf{ z11l{eX^w`vEq|Oi?rEoK5iPsjCRP}cj13?t;XY0rx7lCEZpsusVKC{?!aBY54Q<|R z5f5VAlJoOR8ttTe62t2VE>2u4;)G>ER!G%1Jr@cp6GgodYOo6l%qHo#m>YYw-$Vlg zJ{OQ1f>WwhOHf(I{a2yPG*5Ymy`UhcM>Y^UuhWT88X1I|x?i5TV{qU=f;BuJdjz`? zc_9}_W%YJJUbKHyI-(~8MetxZw~XYhL5*RF)$S*kFT?7>vFp}p*|jioSa|n}Uss<( zRO2zUHBVF^v3~E0%69dVXAn;7TD-gCy<{x&^=bo~(XgJ~hJ2&C}5_BRe!OPrAV|__RXtg#|lIxz{glWYVRk z%k$r=Y4=0svBSzf={GulYA<`N(EsKQed=jeW4c;o)V;`DTB)Uoa5DH|<`Y-f zf*TH$whwSqnJ=^)_uAn|VTUc#l5cEuMTD;@+2f1y&}))e3G#eg^tIpV%`mhUu(lMt zX^NZN<>~#6jP52h+(4j^@Es!X2rhB210zNNqta35Rc+6Mc_%VG)~|yco)8^#zmSj0 zI$-##fM-=)@7bWPE0^=qO8Rhy0Poz8gWszSkqARTcJxJD6r=FFqkiNf*^h2*>$jt5K34y9sC&UKsbn95T|9cAoLWl_OsL&IyVd`X190BJJs5#qdr*``Z zQi*C|vO04+iZ)p{|2P1@HjH;#)3AHD8$Ym8D{FsD?}BSB)}!Su?=Dm_@d{5*!!OH- z6>x-`$Bmjw-6+P-)C}r81OafXdgD^NfU=;a2uD|Z0M{w zNk?7D-$Ikp$A0+TVEn#tM;P6O@u0Grm02NuUpI4!&G*vxkEV5-1~TYK(^jF4;w)52 zKOLPMKRXGs75M~XRF(m)XsuyImmVonkFu?G*zRTpkDvzkssBnO7Sc{!*xtOWsMU?c zew$6f;(q&Bv&=jugnN3psOE9E?dJ!|S3#+R@uo{v)z8(i<>IZjT78U4#t%aLrc zn7Fh|(6|_PxbKI;XZ{27PHL0E!bIWLvC7K#7AgcE<{~R6Lc0)+(en4R?-o7aA7p+o z_O-eRnGrtUAbPI-IRTSTK_aQ)Rs{*`m)$e*4Exl?4-5bIs^cUcbOU${iLr4@P)M{P z;{u*|9j7D4coOjtj%14e{Ttr6$>zYGdpxl&=H7z1cS_xRera(7cG;%q&rPSbKEtXs zbkuYw0#SkQr5<3@-RveCc6{?j{*o0XHFYR{!`W|dlRkS0P%Um>*9^dhBmjjQ0z(6L zYdT@?k{q!yCd0}>Vb?Yiijvsy4Y?(t#T3i%M1+$$9WQOQ@MqwWmwZr9+O;$L9Yg8P*kr7-j_<-Dg zx}7PXNl5ot5FGczSKiNw^*b+mdek1EPTC*EpVS6c=evjcW`v{<72&?EbwaV8RZ)v@ z?Z~F5Q#H-ESQ6-Hb^9oBqli8g0=3%WZ+u~zo}o9|&|evgOuCaGar!auXe7A`$uB$z z(dRTeLD8eH> z1rhY0K=0c8x~&+wV$YXgu-w-JkE~{Ow)Yitj-5#myqak>pon;;o(mg5UE$%qpsg4R z3y&OO-ANwl)X>llp(O3phxL5V6M7;vn3vc&FLQii*mKi{4T6X1z8Ip5;4bonV?^ZB zTvKlKu3)f_b^ zlG}nM$lg2O)!%zrB7A7W7#Sf59elcMq1W@H4V^Kndr9UT$PT9El#soWh_W`upQ1Hg zZK!oiP*UlPjZO>+4SSewx%LV&UGHdn1XGn#y&8#Vt5cynZW5jP$XXUq8#qPS5gUEl z66AE7C+v2(d*1M+d?%r~Rt7OdT9%uwHB>8@&cS0u6nZgb(6-FNyQAnv^kuZ1g2_)Z zxCstaat~Bkqc98^$sO)q+V{++4_8`Wo!lW~CC&%MCig? zDiPmrWaiapj!Ha+?@9#~OFPL=z0g*S-X}|1yM_L0Jr?QKWBA3!d)4_Vys_=aZk;lY z7y}cX)J*&=baS#>?8BAc#E=p=wz&t&`k99WzU5@f2RTYXUKhUv_i4uWk&9zdF@U;m zz#ds<;C~L`(UHl0X9}Tx31WnU+iWcvG3(z>W+~UnQFL5_`Yd5+nh&CHw!li9CH;FR&Sd8o0n)Nj8_`rbT|c6EalL8jCIb{{LnY~LX5F(U9F zO{g{rNd_P3m>7C#-`A6~VNj_v_^>pLOn)&VG-1Y~g}u@piKHs)!%%HtMSuNm~oHc*ai?lv$E~JK}r>GeegPJ}OMynU}dJTkgJT0|rv& zZm_uL4K^NOtDKB;W<00bLHGVBwFfYCB4u?|jks7VOL+67IHf1d6e9asSIL~M zVPi#8)&)gqSUm}Ou7eDxLEI>W2*e_IAJg*+$k*TcUmVSpqw4)%?0scam0i~^Dk_K| z4U*Cg(nv~3hk$fRHyi0jVGBx2cXvuRNOyN`>F#C|XZbwO`+n~k<9uhFGsgLO82q{i z?6~i>=Dg-L=ejPL6MM17wCK)G)A*hKqA9K*DNn4?`O@*CH3{09$*3-_`tolNB6NKl zVo2(0T`^&#WlxIvC-eL-@@5(dPDO-2raSH(SmWj+e0cB-rR4!mgNR=?Hh9gIYX9Xa4gd z7MAJUMYiU;FX^*u67#WycsQN09W=e5K9<3*R=#{Pg!)T@lZJ~V+b27g4;!bEw#Sbz zE(l6IikDGI}!_8v3k@<;)kRr`ey0uJbLa&(mcYaAaT{&VhP#a~@Iz%>rHzwZBM9uUFcVcWm` zm8kzWmtV+EX0iVkya)oqEctU_zd(T-t(ty!zKEEBFi!F3@;C0hb#QF) zZK0TW!`be}=)B^es|Xc}|D~edjwqv{lT&L5KK|Zjk1{c@L!p4@>TnCN4IVe8Gv6v* z>2!B<-P+m)B4+|#r-#EWkG_8W+RzTpaslOsor#=KKRzE-2OIk}3{mei z0sXZIHL}yeRbzp5bxDe|Rr8v2xj2-bo_@@)J6c8lGP>QXN`Xbm z;JqP|KRbPjMoI?lPrh0>eF-}w{XMW|yT5oOWOs=k)23gB+|+^e=WD-0`Y~T&pl@Dz z5h}Ekz-JW(@elDMm-}gTl)hK*#BGSs{V)R2AWVX_yY;7V6F8`UHef6%BC@=}G)F{4 zoF_CfKJgTd#9%h7FVx=Tq^Ye$vwpDDAM32v(;v;SGsF;M@{7;~O3l``85db3(&K)g zii*=gy$c5}uI#(A?8OB z85zs2eLYuif|ri*oTkM2_~k!o*GI*o*8ofem5kcw%F5U>zBlRJFZjNC$D6#Mvi=(WV2cA-XfT1q|q1M+>^#T>z^ZhF~sg6kVF`&+PeOo<-g@uL4_#pWSPFZ#6 z0dQf)Wr@9CcjSRAe`%8l3YMEJ(#5&HR21cIX~6$$UlTl4vq$)xI_tdoih;)YHrhLQ zD_%2x{3)}4x#>kQF#6oRT^Z{B*8PH1NtfR{IVp;&9e(Q(M~)mDJqIARXM=1s+kPA0;>l2?NED zpbTW6zYY|oelfML7zRuGYt^6C?IQ^Zi;ldm_D6nRLx_7_Q{UKFTH5nVSah95`(|g5 z*!c#6^2t7UTFjRmTM@Sgs&%X2Lz?i)7Ua6MULUq-!)^7 z(Q;>2`1I*0BP>jUj10IYU%Bp7k*@a_9j!&1U_GCxM%I7V&HW&Fw~!GW&@{xtT9jj0 zkB%Nc?KG+^)kvi?O8MdogaN!}%HAyRCA0#4EPr;h?y$#!(c|(OiW=tnSVcq z&7~P4gkRr^lNa-vhZ^0k4CdBT?CtFfhJa}8CJ^R7c)GxMaS<1qmj_PSCGk3+6Z}3o zUORDr@BoTVOuPV2sx8zwo$1KR3JW*oX~a|#a^)6SUY=g$-jylE|JS-wx-Wm@@dQm9 zoJ7>t-Ywpn8xzcSVy#v$wWtJ^>L6?=hQ3;Be9yHi{JQen!7pJTcB}OoVq?^<^+4vc z6xqRId9`wmJU9>=8X6iA6H&gmc-wV_!Z};1)ogOMBJsktCl{~R>wfR_nmla=@xQ0J{WMUBwJlS+iBigltMyoaCg4^ zrf6cKgGR*LFLO>ynR#ugib&={CggURT2RDvz-!MOacWP^Eff54{ zm^YPAn!g$A^joj^MDi5zcE zVa~ojAhm;MC01ATL|+~0zfyhg^ekEziTsxqGUnpjcs_K5qp3&%p^8s2C9wI%jf?e= zhnqYXyEynnqxA}U*G5Gbj7T(KAao#WLJuf2n!oct+_5p`HeC#T^5h97W|3gIby!m4 zqXQYmU$;+z2L61Vr@wXeKB~?OnA2#Gk&zMq^#!cTaB6LbyF|PB3VtUfaD8D~s*bAp zy((ldwshtE{@~+B+wPgb7>j-tB_#q*lifw3d*?Mj82Fcq-MNF=p`z~;Rs`wAW)G$y z`uerjU4uJbKzI?A-}ds`(62A3n%dIR7RTe}IYUX!#I^_BM-je$&&`LIi*iY{c&-+Scb*%%->(Z!c>er3K}F4HrnD^}WI+C!Clf=5`vc9Bj_F+ubVUTFE22jT zsn+2Z#i;jH?i-MLXG{@rgT?B|Bxp=Ux|biHP%Jhr+47JtsEV>OLc70JC*Nwv)8|G9 zb{K#Y-8|>EcDA{mknK}d&CWKthZ_o=MBS9xtNEi5 z0|?=|eTtDGesot+R#jS7#R{+Q(|dY1WrX1aR7m$>7t%n?&zpcSfl;?kTUhuAh%G#y z`uXt)6GE!DZ4aOTh*Ti)vg$QZjT=Us-H^X)u0P`ZHD?NhEE^tB<4t*iSX7|K@p~5c zftCnXL|7O;r-_yJZasHNTLNFSs_gTC(h&{xNsTc4NHPS3*OYg7JZ4p6jE3c~1~iDd zA6Y8Q(-&?z z{E>l;gA-Rm$xd;yd}V)PavPSMtm*V6RtolMh~}Z z!)v#;x02xSgLSk_M=W0ht8+#Ucd&(rjl>)|{R1tV9SOOMG-_L?+l|IIS)m=z+;1+u zy>}UCX+JiF?>n7~`lB8g=%?l9$0sK{%?%y&W(78k1Z2=C9C;~#a5s~l)t3td?9c{> zhMaeoDjeZxi}^0iL^g{RM_$7M+vNcPy{9kP-Ou)m((~gZk~~>n|6cvR5`}{^VaQ{9 zi9}{}-hYribXk0l0AcpGzX-Vpk`=2)#*q0Y`{ilHjbwq!N=L873SlV?*Q0P_jTw!d zslvg_(>&HJ)xLdD}MYUjPm? zhZzK!L!^g?7CZ;)P@mr4tlZR=i}d)Qf8t40*xKT&g6zx`#FT!g3+y0goa|vY4GetN zoN{D+7U_Pz2Z1%UO^;A8Fwk$#Nu6c!!`&w|f}mA-heeN@_OIcvvx|+}D@w-GnXw#_ zTlgQ9A3QvE?Vf3g^wG)!o9ZqBf%#Y-#A}CK#E4LVx2hQ85WP z&ATYDun6ch0>xX9@c#%M0l%MWF5vMirJQ}XJtiS4O2NR+&T)DK-os_Bt$}Q!t9`s8 zv&QAVsZkrvAN?4}6g4ZPBnn(~*(~g}ijbb@G?WA~kgZOSw2cqKX(?B)D+_?&mDBFX zO3N)=_MzFi+iD;r<{i?=6nE^&%8kuJ9S$}wiIAN-&^cQ#oEf9`j<{ZW1VfZ1z>MJ8 za^`lk*(U)fv$;$3gzPq1EP6pSAaJmb|IL*4J@_TJpnw1q zud?Zox2#-NfLhoij_P7P&Zb+N^~>+Dn_9?0Rv;cy^KFupOt#RwG$6OAkjOF9G6KBd znFN^jgb#0ld@oQvQntTxlIG};RFIb5Ze1$?lDB=hEX#66rRt`GI^BbVcb^6ssUHlJ zF5uhv=;Kn$$Mf5r;!8>O%~!`bBj6k|euE&$*>0wB>?3AixB_4hU{2RrR$W@Z9+3-E>wZhLjSSl)am@X{3;AlzRuHckXek@LMdRTIvgjf(m4^6hWc7Tt^eC^86AtB1w zZ2UM59AS$z-goahnwX0P5|e~LfHpDaw4suXjERXVodH0Tur5`SqOhb3y&;{7lG1Cj z%zCz(o!4QLmWrte)Ihd}VR}w`(bpTem?GWzl{?}=1X%?I3=|Y&O``s+hF!raJW@3; zS20Cu<_WO)9QXT;G_aJFMY>ONhFp}Bg*>H^SXo)E#V7}qLVbX5*KFm6GmWXK#+x@x zOhrK1MN3nIUEx(bNybQFVIeq1#msg01Z$}ZQL((-`86kdz3n~`WRr}Z97xWUIZ3N< zasrqJB%sMNA2~R(d&hFxw6?>%;ZX!DulmoOgmu{&~=bA(=y9G@RejE{#$M{{wo z#q&GH+YjlEYf~p-%CU|X6c&2mAE#}E&v6-utkd|N_n}A7*XR2=w49ZCT`s`{~*8=e@)ryD&T~bS8Hg*q8K?V7OfFss9?zTb-=5lTy z$L)?wW^@s<2QqE0_xRz}iK#-Kkgu=LG*D(^K2SnaU?~~w>_k2~67sm%`p~kfoHI&qWE|1j{=Dmg_-@iB=oQjlf0>zqmPQ;Ld!- z&6+p~*rQ2GN)~F?N9$-__f0a*Yr@(!AA*1dD(_Ka24HrBwY-x13IaO@p7h7Z=j&8E z2b`bL?cnQEFpteSeUP>g2G}k48%7K}_ZMq#uO}GBv76w0U$d!B1m2NB z8MPb4q@?I*U)y$$(>-cz%$ARL+5G*bjE##k29hJD7v!;k@;ovuEVcMH4Oq$B^CUjK z@Fl=T7#ImYpvE_8X({cBPT@vR@*)) zf7UN2Sid!{C!M#c?x0kq>*;a#T6l+4>TLV5#!6oP{Fo{bWDq=D+~}B?u8yX9m;FRg zqvi~DihLhfT>ODv2AB=#7cx=qH z$wkRv`TlPDU~i?~x=UV0rhrN4J~AbSrtBl3Ie2&g>pu`~0UjD>VBy*EfQ*@DvgUP1 zgnK;=44HsB4`;_%-_BmbR9+rx%~aAbE=5E(`A4P?T?VBfa8nWWg&sgbK&;5g-uYe=9TO82TdjJH%L}`|UL(Y3 z5W+wRZQaxI68`c9+%50|;}ijeJi{ckm=j{Ef! zMFSp3g3QcqexV#(tiZWySs*fbY2_NMugT)VTuE8U-*Vr-hmb?fEO*8wdrZnjNrm$5 z!MdgT;A4>@^U}*4BYJw;8|4iRjXyg1)GzWnud{#%dj%TP4SVkt^ZAu7g%rLaxH-@z z{5f0l{CC1Xu(SYGIVEiNY1q?G~kzZ zde`z^TU1y|R`O%{%sOuL3!j5IK3=dlPi^f20)jRTY?5PvB%T3vMlL62j8}b+qgyic z?MrI(ccKRSeKFbq?A|sXNXdRBN z*kFXMKqwxf@25%p!V2mvdVr7`jI}pGw7woK{6#NC%-G7HH9V|)u($X}c4m#baA^qN zbI&Jez_{?rxNrSiHRM@fKn>AgDw!xQqoxwYOMm5eNapP^C;y^6BS7{k8PU{K8netS z%*DR4-`ZwCe4u!NU}<4~lpQSX>} zUoJx^B3S?uax;l&Sb)3&T(V{!*-&CEMF7-$LeG$0u`%s#Hh-j4+rae)=lSj%eKACMf)zH8Kj1Z&Vsz2R~-OQ!YPg^R{b=tA1Hrpz5W*wQC zx#xNZK*(P?ApDOn-6sb-Ue_D-qy@A#`S02IR{?>Z7p$C>DG6PG^}nvG6GTN}$a7l2 zZ9_nuL2P4;P0&J|zj9NQGBq>=YmqUHME51)t9;QHdUD=A4-kgMzn$0le(`-$Tk8&7 zEAc`1rr8#QjPhop4;VpuN;^C3>YJ!`8S)j-GXUOMMp`Bc8wXxfBb&@t#f z4#1q*dn*<_K^iC{Fu|B^{~#|vSfzFG66UFZEt>?h*s1l@+hy#)z&^R6^D==ai;a)C}bLrnRo{F!fruqVRB~Ukmk> zSR8dp=p)=wl{?%a;7)euMt}kZ><3_LFb*+A7YUCDx5%7t*o*n~JIox$b*Uloa0-cc-VPPOlSuk15&dM1T5uvQ2 zvIo4a%5Vz!uf08|lS4xGS@@hUwst@krlO!QG`0r9z4@uB?^l)?g$Xdpj3(k8*Ku(H z?FIUf+5kqI0yi({3L;Z(`BeKz=a9Qr;#e zEnn}afF3rj9_lz_7NNREt-78e)uU^jk7 zMoGWC6P9A@z5qTelMp04CPuH;gz4-F>c7$hKoB7TK|g=~)UI`7svLWXj~_(BSLGkU z9`zZL(_W2FciQz8)MdEQ(T_Yl02zMp_{oQq4oU=SGp?4`D2q)^`^m}Ao;~vu7eB!R zCu|x#m@C*>sULjXA8u0%{Afe0+qALpE{iVg1E+(3creR7CB~ct5Uz^hZXV)0uFiBOPisHa6{Z)m%FDB)m?E z=t?RoKmtD)bT*GL)ivg1?&x@X39AJq6J=vw-n>9vP0U)IH_q!<|X{OE(rKCW= zp6`!c7+P3pTMe)Ti+y$y5rcg=eSQi$7TvpVCTN;m?sh?y*{@b+)sep)>ht0`=t9k( zpUhRR;E8lkf5Q4OF>Y1w*t~V0a#9YEiICUn`)?T1l0T8iqEw>#DM~^hp(C3CO`!i= zQc#dk@1IaFadz4exT?WuaPa&6O0Dz7!-p0PmBL|uVo0AhN7@#~A}&nL&9{n)`S6(& zB4b^qV6eV9_o1fShVK$+#71}O!O_a0naT2@kQhh6cYg7G%6D$d>vatgL5gXs^%m-0 zXp3<6z#@dR@m z3=Rodw&Rf>6i{Q-5kYc2=h%g7*aP=aJG-VkDD$ZZI`!oqY(&t;0rK4t&F11kh^IEQGvF^F zum)zsa?7DCMSv+CMjkd3PJ!?Tm_v3pw!uW++N+#k&`|g}u7>0TH|`sP==!UD_>c*# zCC2>Djy`>A(CDS7x121xKiwL;^WIki?7f!PwUmg6!NBzys+*;!4Ep&cD;edc?#Cp1 z!Ug$nn+d(3$uyY!CMh=dJrG(3ePd>7lc9YbOVwhtHzjArBx&FC=*ZgdqxEzOmO`aM zM`{9!^Yeiz*HxPDQxNz706jT50R%d%)Y|gnuc=Qd|7|M@zc?jLZf8ndyn)HE{N&`A zWWHHw!#@u7I;7EkDnywk1URdxs(#4K_L@G~*+nqo>IAt1etLPaRz#BY#w9g#Nmm}

H@QX+s59MD47n0_fW*KfNCqU)*DRd2 z+oWtN_BZdO|?K_F@=RhMF%wz~Qc;AC3`1Pw`_WJQ&3pY$6MwH9WEqXd zb^E@rtsRvx(6*T@)IOqQ{D8glG*^FYovG37BJOwqdfz069v=cKfHh0*)>e_JBAdej zw0`&JA1ZP=}CGfdX%>F)=Ycb+p~jw~29Hem@B{F`8@SBU|m(^Zssjo4c7!w(W*J z3&EwXe^BPkNysw1xWtK{$}{l?1-3TmZtHdA)%kx2NJK2%xk+ zLlAg^@D@1ip_wf_ae*`Hl!jk>EZxkkqh<%YV`GlO8dJWhh6LEKkrzp{IC8OW3H^p` z7y!!#Vm`ZA*M}UGiYvA}5!6AJ`?Q(@thAjbLq6{xz6}NFLHheS=ugX02(bWz)<>Zt zBA~yA&h&x9>h>xJ7`4vN&x1UZ+ue;L6Vu>%QYVazM1YCFf#?RR1yzX;L!iU{_m7#u ze@^!NUo|bo`iR%nf0{{sv_(3D9F498OAZ?qTGu^!;(v)oG#(VruTN$vS(KB?jFcX_uRlT^>^fZjSm`tP?`vOxR19uxP&6?nV3mU~Gm zC|K^R|NW&kSx0mS(sD|KK8+eZm5>0X>1jp9)0T^&ihhGzp(gLrzRga9rN4@{KFO(yA zuz2FGHfN(JbTIAJcVKcB1;YPqoo5$_JD_U{I$n$N$|sJO1;lezURTFoW5o4f9|uEE z>K%7f->bXabnAnTSzNh^Lc)#Qw*QM(dVlIy=QwiB$iC-$rb9`lb4BjJ3}>Xxo0A-c zA)47nd(m|>fgHv^YV=-L_pWn~5BLRuQ5t0}t&I&i!pO)dJQj`UsH94-f$boL6vUvv zN66lO6XWUEAI}?uxL+{P(J6Wp5WJL2=4GOK^9IZwkcnw(x^0xW^AZv&+dKZn@6V46 z+pg4Aipr*e2p9U4(V?; zf7G!+O|d*OGSX@9N=HT}5IvoWrrKrZxa$S!vrl7kqPq<7URWT?G~YETh>H56lhe>t zJ6!et9PSe7(Tvbn@@ta_QBqX}r-&1p-vcBBq|r6r8PA@+1k*#~R-YdI^^nKOTlGV} z2F3ty>&K6uB)@SyR_>$$G&>+OdV8nGwg1f{^HoD+;Flxwtp->56CeyQ6&0puq$9%W zst?{imFSvUS^_;W3MwiM(5*O*%v)@FBp?8X!22x7#qzQN3nT?3Wek2j)ORrKt&{G|=YVCND$x-* zfop&sc&+XJJjl9Ld(-BVS%I7OPyXI#2JQpw#LZTH-QBc|j13M;H+_?;-#z-m&@u0}jio*v6~wS0kqssL{V}K& z7XyomTW|dDt?P&m2DpLFJ2Y%VAq9Ai@M4*nTY`B&N@~i(>~=8;43UaTM=0i7H~?O2 ztHrS65>_2$nFxJoD<&af8$2;00*pIF#N^zKkBszOp3(`igvPvL=9(<^kC0r!D6V=x za+)waIr^u^`ZM)2x4`*A{q`e7CQ=w1IMFsmS~{}}h&2H8<6q1E{ekW@=Mz+G#43Ez z!jZ&uK*{v@hwk zlY{39k_SbmEsqKmozC1m9)W;~oqcS)Usgpf*jZSjz08`CqN2dq zS3WQjbU%{Dv$u~A;>fr@?yi3R_c)6tisCJPLPP{egFryP*6vJ`B10bJ5itLe(G{|{ zv?@#Ga&-)3!bV1oJE^nn?Ch!*>V_`&-GK<{27$+QNOa>nT=3I9nMP^R8jx4%nV1?4CM+I3dIZ`O`fU=^ zGkhEd0twZ(nD|6&dvh<-W!{tOt2nG!#BoT0?Xe61KH!c6W0cNoBQI6^8p$uYj1yH-xpDTS9{X0 z&~Yz#C$M{8zvHYaP*3;CusQKVY10e-dkf`L_Hp<8Y; zXy4L&GwJI765TTV4OEsEXQ4)u8p?8$@o5bu{Ac8rJ#og%7Z>s&zoFOm3v6< zHAjPcx3)I`XMYW>VeRIIAX6cFv5bR1=_X?&sEz+#O2hQu4kxcyGcz^VP3MoAZ{N`5 zfO@tKe&J1gsi(eaXki2HFEC@evC$qH7%2MVDA~Yh2K@0QS zmIa~pzaO}I4%S@lJOHze09^zPP`9f7H|SLIkE>uc6|YF%X6FQ(IXDjk6Z6ttY$e?g z@IJ2qUA4rBa!WXiA_fL3V=J|i9!cDv(G>9Ww&0hND&8Op%UfDia3XYjoom|k?GC}# z-`fts47vSA1%nCpzs6B2U<}J{PQZ~Pq~XY*uf-eQyfQ@ zWwj09>_8v{j3k=(Sy`>`KSS4MQU5VqEdqu7d-}^v8vox&iuH^BzYJf{vHt(%zi#mV z$vsgTAV%Ow?s^hhY-p7`^dlzy;pCI?n9*PI-u-#34|X0>u!wVgdfxAg*d(SFDq<=$ zuxM-!Tk(}&rCwu`V%c^w81FK>Y|-b&yX=4k>h7DCQZ(4dUJbiZ@}D!c>g^uKIduj*L~z9TYaC9D4_F~UXlK) z@$h!#f+e(xv6m53>A?+QT3odT9w|GqWm0;7dhq$h0( z&}ISBme*hQYKlW(Z=`!A#dQOo%@KCyEhBNee4a#7v-ju-H=20<=k%SVfs5Blb>2Nl^3!Re(2O97XLaqJrKcxTG*jzOb zL#lm!W-!~wVS}TFZJEm=d3wVyH;k<%W@(D?M&(V0ptxV-^OAw4WTNo?gpBl(^oe(Y zI`2)!rzAzBeUz^b@EnZ=){yZ!8O&(kd*>786%ghT);q2TV;{=>3ae;v;4=(%>LiG6 zV=VD-rlKIvz|fF=F5Mt_A-=ao@gn&BL8g3p>W27>f+f{))8Ih@5<>pW)hs?bE1FW~onD7)bi~^0zGVBPf*1Dq z_0HMhXIfjcX%g^@G*XvxqHI1|Gg?YSO6WdnpTdCJxGKlbG42yq!{k)A=Z=0|May?4 zM%LFyBl{A&K5UE9s~?P# zHh2i@2^q}jvAyVJ>rzg4OEz}-x!UB0S#*8d9N-P(Hx%HcA#0Q2V*=ZqBzj zH5A#X>Ac~ry&1phmQh^FX)U;?7S{`HsBh0QqH-VH%gZggJeuj^E7*_rU`~4x>I${9 zuxrFS<#O6#EMeA*RXiO8LV!Zaswy9jCNoTOCm~-g;Us%~&w5D?kc&#AW-m&$Ld*7> zdroe+H^y|cPEfSpPyhma77t;$y zyA_D+TRLGV$cx!Zg+0+t@RaM*?z0IX)-MhB!6dbMk*3KeUfFIViZ|KgiAtR7yU8OTP#73e7zk~`ZR39+8nxTo-`x+c-G&%I(cWS3gw>su( zN^-`n;JaX`G_N#8XwWK^jp^v1V{M>?`vg4UfZL3LIB|9^q@0RT1c`OqaAT8Y0goq4G$W#O;4ya|9axcu@*%3@A1I-ceKIemccWbNQvJ6l zs3Wr_mBl=!hN|N)aYcN~hD{YA`M?U^t=}Q2mZa_Qi>qo|k?eKC`xGXBjeKV%A?_po zn6@>$njd%+?gIx|6Ym)KxDupOXN6T@^7m^aF`N(a&o_qUi<5E_qIc$>!4S`uw2w26l~``q7;v zo#PwKkLt~0%@SYUI-m8O(FN0XQuBBO#{bqpm#*%wrmJGN<~|?C-)@P3hJn~Cu)*UU za<9E|R1v(0CwWW!u&rb3NT3}{pW`fEoe}ryVy>pJtniC(%gz{tekO`E#d~*m@y+Ij zH#{P}pFA2-MqaMCX~XzXZumXxlp*i=uGz;NmUewKH%(kkXjr7gfvd6-N!GO7ez9@! zJ5>8&%5TktLLURwPA6hV8a(;BA*bBpF7(2~atbrXXNt;= zDhnDmxm%o%wuzb?X$P2s>j)Oo%U)sQWouL3^Xt~1Yqy-x*6hX>?wr`1>Wo(J;3vdL z=n3A!%9oz{n`>DB!4|NNAS5tb~1(X>5 z)aP~gQyhe(2Hsn5-jRX8*hkwNi5G)HlGgFjetNqgfK*Qspse{=&7GjJ%M<>*Ht%pGvDhNCnGBJJ^6h+xf}+ohLGo<^R??5Vah+) zPgv_KSI^wbC)l01<&k7xLB(`K?2>n^ z-t;fG02@{kEwoZ_4XI=U!`9m>{$F7$Kyv3tQ(_!88;m@Cd@B6PCS6$KaCTHL%lU{^ zc|?s)@_tC=OC49;xHkD=%MWswcZBC$4UPRmbUacTYOcRJ?-vegY;4QBr_Z~M=1n28FE)x=mHhPCPaZ<{s+6QC>jPU*Sdef&V8{YkD zlPX+C9K}XzstTfG1?+U>@n~P9w6b7`Fj0OCapjz!lvc=)zhOLNW_n{M)=kA#g{q#7=I&$zI;66u;+l6=A`fM6o(Z~I)?C;0 zYvlDy@zPm|9pBY3L%v*Bv?Xm-0=ZuZ>h%gr>7N;i6P?v8tv39j@JFGF8kQ$XAX^~w zP)(wP(085psv4?sFXOI`HyNbNrR3NLo>)_@YxdVC9P|eEm8X}Jv%D*%)X3jc`HHFE zr3*a9KXZNoYZ8YFCr?3{bk7T@>+DYakp4JbU}&tdwlMMFvH9a&pE9*By+-ewrerzJ z@V^%0k&Y0|Vp$8RzdW@t$K!tE_q0~&F29qorZ9j5t_LUh>qmQ6-@wJSjtmD*;VBRH>ttV(Nz~qk6vs{+UHG_H1q3z>S@x)GcV}?$Nr!% z&J}~RjtT`|-_3=WOl(TrAAQt1eXn)%@wL-66kUf<#|GzOY_5}E^{2d#n!{>Vt}{0* z3NOlA{*tI>iR3D$xv2P4$26ipL=lT0#mBkNBOyIJB)loft8*@BuJ2Ze#sjQ1FR82J zo8iE1g``(LXWP&!QFNlemz#6)O2ccn7N72`u>;w1B9opq+qUytc|3Dee>U-V%@*IodFs(L5Vzz|VY6g7pBekg|7oDvX7 zON-~{rw7=6hav`2wHQcAzt5>xvQv`9Y!JM?ZvP$8qD7kP_#wf?+$XtcKahe2&6TJQ z0QF9u)B%gC1l952@vPO%b^a6`!@BSg%(E|jON(8DRkw})EL7+z3bJOcR&@wpzx6B#>=llv+3tc+a=suX&L(UABOS207Y z>@$W*mx)@W>Za2~a6J}ey6$U_l0(kMaZT2_yc|MzS~4P81qCMk7F2Z;LS;%gBkPT4lNbtv_h2ET?4 zw-*b=b#yP3byK{Oa3U5USv#o$zR$)`0PMU%KTo?Gc=)Xx_aYldD$kqc_4nE%W;&i) zU;C#;%$i(=R{WFhh{Z30>M-TXT^wa`)mY!No^ZL_pUa(PeC+EH2~-gXeYkRb2RD5f zLv@t0K~s5Wk=~clfsS%u{m6o)E!;|-Oijo+$7aU5TQv7_N<;@`!dvqK_O2ExgOqpo zfl>cfax^u2`WyLjQ8VjAd*MS1)3n2g*Ui&vEaw+#2PmV@3jH*w7Zl3yEqVq`X+F58 zLfDsaL?TY($YG?9=U(nm&x2E%iE(&8UNLEeA=RqP2UWkU({ZHAq@2AlyWjoAKK8&Q z&0WN?&~fpK)vbnFG(C5778d<=jtuGSaDgAOSEQ}pm|E_~fhZeP>-%amu}|&?zIK$w zadSU>aM!~WBUznxv{xEiL1uVteqVb~n4eD1z{|kEyR&izDD=$5QnujleHd`EVZLh)61`vceE|JgJ!eOsEGUB%l*?a52I`-~J zs)#W+er&9b1^@_QC@P5v?0p7~3HG-0)EL*)V8GS2Whf}2EUu>zQ=V1ySAs0)>fE-i zx%EL$)YzPgOe}AEa#v~IyI&+42*939Bz@A5a9R4d?EzujBVm&5K=i4}qLIL9)h&mGf0IxZTTDYh4= zN-#zCt!@arnAhE_rTkT)O|D zx}17PG@-M)K_F>p3`NJebz?DKO8A@%02E+y`YJk^7LjIHXFB&g{XCyj3`)FL&V33q zY3^H3_^j2j))?zY)xvhpAtr&yqKrTtFzzCzS`J6QPDWp@?NRR2meJI$^9()r#66>i9-*e zNT1!;HU~5XBnA{NVDl~d?$C|Pbyuy_SFxMZWKbG%&i9_C_;&&x?C*!&bG-#1Dj5<3 zO4}V#-D@}DlB>q;L`(MB>_7I9oz zs=-&FqRE<69Po;7RMevoM#&;*BKCP@9d$saTcMs=p1@k4Y85(g(5!hUe#^)Eo0bfE zftkLD;&ygjig)(JlI{(ifL7xJttaRr$M6+7yTPT z1o#G@UxmcCx`#Tic1S9-L@+n$)LlB|HAxa2#Ngrrd^B>{Jf!aCd_r8kEtIi#X5-Vc zyg=;~5gy^@6}z8jG7$H;6X2}cqJ8eAR>bWgUuJKd4Qwm`@SUA?mU?A6-P5k;PA~ug zhb2#P)$ukWwE;T9Ue^KicCbjpKK}mrdjFtQ?|wa;+Y~k(QoTm?Q9U#r%YH@Y0#di5 zZtiXGN)9m)dHe9-6>k$X|O)CQ@MZ5Wg$)MO9*g1`|V3)>vcw&-4l{-Lv zZ_Zb2xo15w6&B0g(Yf@84r#AO7fX8F3u=faK2H~K6LDk34RQXCnU0>89#A#vDK&j) zhG~5xXcpqS7?jlPzoVUXGkQPB@wLDD8-99$gbCA^&YY=~8X_B-Dft*vU9vyew1#m{ zD@^xoBSh)YH{)j}zx2_}KInq*aq*oy5k)+{GD$P%C3@hyn)lbr5ZR1=$o+VB79G#>}!;cW2KXF!*W!j=eZ1@ zuxIHsLo4jA((}Dj zA=OYf6Q;L>l*?KiwX5q7n?r|$CcXHc00g-;(lGYuj&wkaOKEgtfhH=9#k&Qpl5=MiBY0LRtI{q!Wg{^Bg4E(3IXAVW>4 zLN9eQax4)Z9C!2kbv~w(pZJh+sJ!&TSpbx?Y8t9nZjhoGHqxc14!9ml;y13p=LG3fIwyRPk_@d%HLG{vO9!P!O)Jq8oJ~ zI0RW@S`1va$Sp~F4G9QtbH4TX6F>D$s7XYo;kbBQ056d88K&0$|v#m*o;<}$f@u8LP1mqSsz(%aTQKsDMU zR1%IO4O+UMOw=<#z!Dj?zelR}N{H^~jjvm`U1v7$PM5 zZun&alma)(TQN%VR5^#r`k*9e?n-1Dyt}q1fFV9tDPypYuxnFj6nwM$VJztQ176MD z;+RkhsLl%WbFag1sLYh#2PepZDiP7TK}}KV&idD>V4xdJd-H-SYfCr#$#DA5$cddM zQQh4c`Jaw@D^6Q;egC7k>x^nLOT(x*0z*e+iHaab1_5DQLIez801*MDNl`~sh7KVb z1R|ZGj6n#}2_-lQ5SmC6V-%^<0z@G6BBHc_2tsIra|yFMyL)zj?4H^A&i#?^vkJoflxD zYEweF+G(2H8XL1Ka33BnliX-QU_w-Gl167mh591m+dZvw zP>wmYi!1oFX^(~4XEoM4`yBfsiv-)HJ<@7ocXPhc6fkB^yb<%j>NQ@(4k>A{3TPBwamc$&ChkpeOPyyNe-8sWwuQMBhlLs zyCQ1FL1U^iq}S1|>r`)P+ARZ{LULC9zCqpmAneC~%S=!&U%S3rUEFq3y&WuJJ_27j zkuUzE9Abl|?kTW@;gyN&j1%mdQ;M*mlv~S#;Gf^vzkEZ4mh~W6r_Hozl&_bK7)TaR zK#YhD{Olg{H{zuFc{SKtnUj#^?;(NcK5*2Tm&PF`*_Pc_Ush+wGcNlWX(g-JQeA?? zkmvj6Zvs+|SB7^NuV;HzMcU9X1#gxfbf}JJnzhipCCB{AuPuq{quI+PUj#`ty;YJk z6Ucg}=#QKV;2?xpZriMzz640Jqb6vuPss}5onLds?mf<1FOBz*`4&Xi%WdKsfJr;P zX+r4$lp9kskC^70**YeQOz=uLCnecz)Zb3dVBL{mvh!Eqqmw2}{nPf{jX$p!qR1%i zwoGBs(s7x1s-j^JLp$OxY&G(hCByWIq&>+a8-uE)=Hq}ZPKC7vV>6X%ubTm9O05788q*_Yxn<8js{E=e{ zECoN$!&y!RPP3?V;S`-VaMF$~PxnqMcb#a>A4q+A5QGDc5AODDsmxs11>(oL>k&jT ztqrB;i7PUfYz^|tt~xabp=en|Bc2ken+|=0D$6c2bs0k1a_?K*npV=s-C?yY{;}BZ zL}1@E{G%pVT=iM?Z;OOlzHL>Eyh=TZadG}e*_e8s=ldsJM=-9&0WSXuA^!Y|rDtaJ z@M?q7s^!}=jr}(X0o4P+g+BdZ!cdQfh}X@rSN1V4o;;_Ff0_SXURKU0B`41EG_#j4 zV5$1>b4hFE45uE4%JwthMW*ng7aAg4HwO2(;vPqTT*Zu!qz4=T9&DC#DZBk53cNtTYu_EoM69R1}99C0UyUAD4v|2U} z7?CaJys##?k5EEZP!K+A@9WRFKSUn-#vutZL7C4<1O_zRn_rei6EiV>8XevmurFTT z?rNjVi_aIM36BP@(*j;jESD1jwt7-!&SBtz-{|YL#xO{;t_=5> z{-P!CovGMb66Y4sU<%NOayiSN0+=tc>{jWsLyt=zpYp^R2F0Vq+$$8zKO{Kh`DyUT zoZ{=;_TKXI97llk8F2h*TPF~3`IXIap>WA?;|WpZ@93HAG3ep!;v-&(0K@RL2Tscm znvCnTB5GUwmYIuM5)R@bNV0f#s-n%mVmHg}2^YMx89ycT#vZ{s>)D>|U&g|>YdEXU zQ8WGw$!vRr6x%V%8blr-tgu~rwCf}f=qkRb6rjk4kh$kE^k^T$n&!jln$CCM019Ee zWJDGnaHBiX`8b9pD-SiBKV$V0!(BrI231Hza@d$Rt}MznPEj0@7`n9E6^9GqlOcas zVDn7m#PoO3XU>FX*m*llB+k4shgi3h_Z~XUkP$9vLSAhm0r$9mA#lyO^avWZkE?wD zb2W{5R250fE!C~$m`!-9em7B#RM;BTx@kRR=$C3`W%~L9nif;C(D&BNUI6S7VE+N- z=0%OQ(e?PXzIlXT*@kJil2y}9BXin|?F#Aryv+ddaL!8KgMp zn+;tEE>{FAd_fcr!A`@9bGsCC!>Ysf4I*2jwy@mtQ9PUdhjs)jxj8m}Cz$hm>N~q; ze^H-qfkTN#DEcday7xC%!~?b3Md+k*r7kQGc6f)v%armxaOJDpZC0UyhtZtKzDNL2 z``_)Uz;zOOZwm)jRKvgJ^g?JI^AMLbH)yr3VA=aks>;1xz%16>vU-4PLIHbZeZ(hM zXubnPjjKV|%S1Vu;-`5ir(aLfj{@od!k)1S2nS=lu|7u#JShVhMw6Gu zAc38JH<*17GO~(Xo$gh$_b@(V0t$so^e541#DmnycGMl8Uh&5-v^P8#XJ0kP3oo+$ z>t712q!j0Bsh6m~u_giPGNy*y@dYhXPMrGe6hqSY8nqV;<-l*jJ@;QM__p z#o+9fnh%o6v=~;tQvpCYEMl#MG2_Dp=!0O{>mAd-sp z8%J;?(S(VlOrqDL(F?t7!MXmu4ggX7(ET2wN`J$>yLVLKi>0$V-TbV@qqp{FSDQb9 zyjsnH%m23%vm4#T$EB)0lauIHRt()M2lr9r3^py# t=l^N;Vt;1f$!YQ*WQ+Dce|P=YU5=*>xW}h@#<#!6Q2(OdW8{s{UjfdzpcnuE literal 0 HcmV?d00001 diff --git a/litellm/main.py b/litellm/main.py index 0553cf9d422..66d69e9fb52 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1,3 +1,5 @@ +# LiteLLM main module: public completion, embedding, streaming, and moderation entrypoints. +# # +-----------------------------------------------+ # | | # | Give Feedback / Get Help | diff --git a/tests/test_litellm/test_main_module_header.py b/tests/test_litellm/test_main_module_header.py new file mode 100644 index 00000000000..a16e14e8c32 --- /dev/null +++ b/tests/test_litellm/test_main_module_header.py @@ -0,0 +1,13 @@ +from pathlib import Path + + +def test_main_py_starts_with_brief_file_description(): + repo_root = Path(__file__).resolve().parents[2] + main_py = repo_root / "litellm" / "main.py" + + first_two_lines = main_py.read_text(encoding="utf-8").splitlines()[:2] + + assert any( + "LiteLLM main module" in line and "entrypoints" in line + for line in first_two_lines + ) From b631863b13abe81c8de78a0787f3520d52427215 Mon Sep 17 00:00:00 2001 From: oss-agent-shin <279349115+oss-agent-shin@users.noreply.github.com> Date: Wed, 6 May 2026 00:42:49 +0000 Subject: [PATCH 19/85] Add utils module docstring Co-authored-by: ishaan-berri --- litellm/utils.py | 2 ++ tests/test_litellm/test_utils_module_docstring.py | 11 +++++++++++ 2 files changed, 13 insertions(+) create mode 100644 tests/test_litellm/test_utils_module_docstring.py diff --git a/litellm/utils.py b/litellm/utils.py index 019fbc2add8..5589852ce41 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1,3 +1,5 @@ +"""Utility helpers for LiteLLM core request handling and provider support.""" + # from __future__ import annotations must be the first non-comment statement from __future__ import annotations diff --git a/tests/test_litellm/test_utils_module_docstring.py b/tests/test_litellm/test_utils_module_docstring.py new file mode 100644 index 00000000000..ac99fb63fd4 --- /dev/null +++ b/tests/test_litellm/test_utils_module_docstring.py @@ -0,0 +1,11 @@ +import ast +from pathlib import Path + + +def test_utils_module_has_docstring(): + utils_path = Path(__file__).parents[2] / "litellm" / "utils.py" + module = ast.parse(utils_path.read_text()) + + assert ast.get_docstring(module) == ( + "Utility helpers for LiteLLM core request handling and provider support." + ) From 5d7b7e7e373ab116a94aba655af28253a2b4282a Mon Sep 17 00:00:00 2001 From: Michael Riad Zaky Date: Wed, 6 May 2026 12:32:30 -0700 Subject: [PATCH 20/85] fix(realtime): register /openai/v1/realtime as websocket route --- litellm/proxy/_types.py | 2 ++ litellm/proxy/proxy_server.py | 1 + litellm/types/utils.py | 1 + tests/test_litellm/proxy/test_proxy_server.py | 34 +++++++++++++++++++ 4 files changed, 38 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c6653a722d6..d777d34eaff 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -353,8 +353,10 @@ class LiteLLMRoutes(enum.Enum): # realtime "/realtime", "/v1/realtime", + "/openai/v1/realtime", "/realtime?{model}", "/v1/realtime?{model}", + "/openai/v1/realtime?{model}", # responses API "/responses", "/v1/responses", diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a5905765c6e..3f7745a54c0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -8805,6 +8805,7 @@ def _realtime_query_params_template( return tuple(params) +@app.websocket("/openai/v1/realtime") @app.websocket("/v1/realtime") @app.websocket("/realtime") async def realtime_websocket_endpoint( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 00a7748309b..400edcac889 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -825,6 +825,7 @@ API_ROUTE_TO_CALL_TYPES = { # Realtime API "/realtime": [CallTypes.arealtime], "/v1/realtime": [CallTypes.arealtime], + "/openai/v1/realtime": [CallTypes.arealtime], # Provider-specific routes "/anthropic/v1/messages": [CallTypes.anthropic_messages], # Google GenAI routes diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 3f19db36c3f..6718f52cbf1 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -6513,3 +6513,37 @@ async def test_get_current_spend_redis_error_falls_back_to_in_memory(): finally: ps.spend_counter_cache = orig_counter ps.prisma_client = orig_prisma + + +def test_realtime_websocket_route_aliases_registered(): + """Realtime sessions reach the proxy via three path aliases stacked on + `realtime_websocket_endpoint`. Dropping any of them silently 405s + WebSocket upgrades because the catch-all `/openai/{endpoint:path}` + HTTP passthrough only declares HTTP methods. The aliases must also be + in `LiteLLMRoutes.openai_routes` (so non-admin / team / key-scoped + auth allows them) and in `API_ROUTE_TO_CALL_TYPES` (so call-type-aware + logic such as guardrails can resolve the realtime call type).""" + from starlette.routing import WebSocketRoute + + from litellm.proxy._types import LiteLLMRoutes + from litellm.proxy.proxy_server import app + from litellm.types.utils import API_ROUTE_TO_CALL_TYPES, CallTypes + + websocket_paths = { + route.path for route in app.routes if isinstance(route, WebSocketRoute) + } + openai_routes = LiteLLMRoutes.openai_routes.value + + for expected in ("/openai/v1/realtime", "/v1/realtime", "/realtime"): + assert expected in websocket_paths, ( + f"{expected!r} missing from registered WebSocket routes; the " + f"realtime endpoint will 405 for clients hitting this path." + ) + assert expected in openai_routes, ( + f"{expected!r} missing from LiteLLMRoutes.openai_routes; " + f"non-admin / team / key-scoped users will get 403 on this path." + ) + assert API_ROUTE_TO_CALL_TYPES.get(expected) == [CallTypes.arealtime], ( + f"{expected!r} missing from API_ROUTE_TO_CALL_TYPES; call-type " + f"resolution will return None and break call-type-aware features." + ) From 32b031aa2cc13c9e06c6ac9fc40940c0c9514e97 Mon Sep 17 00:00:00 2001 From: Milan Date: Thu, 7 May 2026 01:20:03 +0300 Subject: [PATCH 21/85] Flush virtual-key model max budget increments to Redis after success logging. _PROXY_VirtualKeyModelMaxBudgetLimiter subclasses RouterBudgetLimiting but does not run its __init__, so the periodic Redis sync task never starts and spend stayed in memory. Push the increment pipeline when Redis is configured so multi-worker enforcement and cache keys stay consistent. Co-authored-by: Cursor --- litellm/proxy/hooks/model_max_budget_limiter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index 95ffafb7bad..9286424878c 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -319,6 +319,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): response_cost=response_cost, ) + if self.dual_cache.redis_cache is not None: + await self._push_in_memory_increments_to_redis() + verbose_proxy_logger.debug( "current state of in memory cache %s", json.dumps( From fa4c7a2ac6948cbcc975611231cea1608bf04b3b Mon Sep 17 00:00:00 2001 From: Milan Date: Thu, 7 May 2026 01:21:19 +0300 Subject: [PATCH 22/85] Add unit tests for virtual-key model max budget Redis flush. Assert _push_in_memory_increments_to_redis runs after async_log_success_event when dual_cache.redis_cache is set, and is skipped when Redis is not configured. Co-authored-by: Cursor --- ...test_unit_test_max_model_budget_limiter.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py index b4aac113f57..0daa5b17ffa 100644 --- a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py +++ b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py @@ -413,3 +413,71 @@ async def test_async_log_success_event_uses_end_user_model_budget_duration( f"{END_USER_SPEND_CACHE_KEY_PREFIX}:{end_user_id}:{model}:{budget_duration}" ) assert call_kwargs["response_cost"] == 0.05 + + +@pytest.mark.asyncio +async def test_async_log_success_event_pushes_redis_increments_when_redis_configured(): + """ + Virtual-key model max budget limiter does not run RouterBudgetLimiting.__init__, + so the periodic Redis flush task never starts. After logging spend we must call + _push_in_memory_increments_to_redis when Redis is wired so other workers see spend. + """ + dual_cache = DualCache() + dual_cache.redis_cache = object() # truthy placeholder; push only checks is not None + limiter = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=dual_cache) + model = "gpt-4" + kwargs = { + "standard_logging_object": { + "response_cost": 0.01, + "model": model, + "metadata": {"user_api_key_hash": "vk-hash"}, + }, + "litellm_params": { + "metadata": { + "user_api_key_model_max_budget": { + model: {"budget_limit": 10.0, "time_period": "1d"}, + }, + }, + }, + } + with patch.object(limiter, "_increment_spend_for_key", new_callable=AsyncMock): + with patch.object( + limiter, + "_push_in_memory_increments_to_redis", + new_callable=AsyncMock, + ) as mock_push: + await limiter.async_log_success_event( + kwargs, response_obj=None, start_time=None, end_time=None + ) + mock_push.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_log_success_event_skips_redis_push_without_redis(budget_limiter): + """When dual_cache has no Redis backend, do not await _push_in_memory_increments_to_redis.""" + assert budget_limiter.dual_cache.redis_cache is None + model = "gpt-4" + kwargs = { + "standard_logging_object": { + "response_cost": 0.01, + "model": model, + "metadata": {"user_api_key_hash": "vk-hash"}, + }, + "litellm_params": { + "metadata": { + "user_api_key_model_max_budget": { + model: {"budget_limit": 10.0, "time_period": "1d"}, + }, + }, + }, + } + with patch.object(budget_limiter, "_increment_spend_for_key", new_callable=AsyncMock): + with patch.object( + budget_limiter, + "_push_in_memory_increments_to_redis", + new_callable=AsyncMock, + ) as mock_push: + await budget_limiter.async_log_success_event( + kwargs, response_obj=None, start_time=None, end_time=None + ) + mock_push.assert_not_awaited() From 581ae1443d3e80af03030641fe7674e8e12d6c65 Mon Sep 17 00:00:00 2001 From: Michael Riad Zaky Date: Thu, 7 May 2026 09:44:39 -0700 Subject: [PATCH 23/85] honor OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT --- litellm/integrations/opentelemetry.py | 91 ++++++++++++-- .../integrations/test_opentelemetry.py | 111 ++++++++++++++++++ 2 files changed, 193 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 77833e5de0f..e49f0fabdaa 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -57,6 +57,17 @@ LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request" RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request" LITELLM_REQUEST_SPAN_NAME = "litellm_request" +CAPTURE_MODE_NO_CONTENT = "NO_CONTENT" +CAPTURE_MODE_SPAN_ONLY = "SPAN_ONLY" +CAPTURE_MODE_EVENT_ONLY = "EVENT_ONLY" +CAPTURE_MODE_SPAN_AND_EVENT = "SPAN_AND_EVENT" +_VALID_CAPTURE_MODES = { + CAPTURE_MODE_NO_CONTENT, + CAPTURE_MODE_SPAN_ONLY, + CAPTURE_MODE_EVENT_ONLY, + CAPTURE_MODE_SPAN_AND_EVENT, +} + @dataclass class OpenTelemetryConfig: @@ -71,6 +82,9 @@ class OpenTelemetryConfig: ignore_context_propagation: Optional[bool] = None # When True, create a private TracerProvider instead of reusing or setting the global one. skip_set_global: bool = False + # Programmatic override for OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT. + # One of NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT (or "true" as legacy alias). + capture_message_content: Optional[str] = None def __post_init__(self) -> None: # If endpoint is specified but exporter is still the default "console", @@ -182,6 +196,9 @@ class OpenTelemetry(CustomLogger): super().__init__(**kwargs) self._init_metrics(meter_provider) self._init_logs(logger_provider) + # Sample env-var / config / message_logging at init so subsequent + # _capture_in_span / _capture_in_event calls are deterministic. + self._capture_mode_cached = self._compute_capture_mode_from_init_state() self._init_otel_logger_on_litellm_proxy() @staticmethod @@ -306,6 +323,59 @@ class OpenTelemetry(CustomLogger): hasattr(self, "callback_name") and self.callback_name == "langfuse_otel" ) + def _compute_capture_mode_from_init_state(self) -> Optional[str]: + """Sample explicit settings at init. Returns the resolved mode or + None if nothing explicit is set (in which case the legacy + ``self.message_logging`` flag is consulted dynamically per request). + + ``"true"``/``"1"`` map to ``EVENT_ONLY`` per the contrib convention. + Unknown values are ignored. + """ + explicit = self.config.capture_message_content or os.getenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT" + ) + if not explicit: + return None + normalized = explicit.upper() + if normalized in ("TRUE", "1"): + return CAPTURE_MODE_EVENT_ONLY + if normalized in _VALID_CAPTURE_MODES: + return normalized + return None + + def _resolve_capture_mode(self) -> str: + """Return the active capture mode for this request. + + Precedence: + 1. ``litellm.turn_off_message_logging=True`` forces ``NO_CONTENT`` + (kill-switch checked dynamically). + 2. Explicit setting sampled at init from + ``OpenTelemetryConfig.capture_message_content`` or + ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT``. + 3. Legacy ``self.message_logging`` (checked dynamically). + """ + if litellm.turn_off_message_logging: + return CAPTURE_MODE_NO_CONTENT + if self._capture_mode_cached is not None: + return self._capture_mode_cached + return ( + CAPTURE_MODE_SPAN_AND_EVENT + if self.message_logging + else CAPTURE_MODE_NO_CONTENT + ) + + def _capture_in_span(self) -> bool: + return self._resolve_capture_mode() in ( + CAPTURE_MODE_SPAN_ONLY, + CAPTURE_MODE_SPAN_AND_EVENT, + ) + + def _capture_in_event(self) -> bool: + return self._resolve_capture_mode() in ( + CAPTURE_MODE_EVENT_ONLY, + CAPTURE_MODE_SPAN_AND_EVENT, + ) + def _init_tracing(self, tracer_provider): from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider @@ -825,8 +895,7 @@ class OpenTelemetry(CustomLogger): from opentelemetry import trace from opentelemetry.trace import Status, StatusCode - # only log raw LLM request/response if message_logging is on and not globally turned off - if litellm.turn_off_message_logging or not self.message_logging: + if not self._capture_in_span(): return litellm_params = kwargs.get("litellm_params", {}) @@ -1117,9 +1186,14 @@ class OpenTelemetry(CustomLogger): } if role == "tool" and msg.get("id"): attrs["id"] = msg["id"] - if self.message_logging and msg.get("content"): + capture_event_content = self._capture_in_event() + if capture_event_content and msg.get("content"): attrs["gen_ai.prompt"] = msg["content"] + body = msg.copy() + if not capture_event_content: + body.pop("content", None) + log_record = SdkLogRecord( timestamp=self._to_ns(datetime.now()), trace_id=parent_ctx.trace_id, @@ -1127,7 +1201,7 @@ class OpenTelemetry(CustomLogger): trace_flags=parent_ctx.trace_flags, severity_number=SeverityNumber.INFO, severity_text="INFO", - body=msg.copy(), + body=body, attributes=attrs, ) otel_logger.emit(log_record) @@ -1141,14 +1215,15 @@ class OpenTelemetry(CustomLogger): "finish_reason": choice.get("finish_reason"), } body_msg = choice.get("message", {}) - if self.message_logging and body_msg.get("content"): + capture_event_content = self._capture_in_event() + if capture_event_content and body_msg.get("content"): attrs["message.content"] = body_msg["content"] body = { "index": idx, "finish_reason": choice.get("finish_reason"), "message": {"role": body_msg.get("role", "assistant")}, } - if self.message_logging and body_msg.get("content"): + if capture_event_content and body_msg.get("content"): body["message"]["content"] = body_msg["content"] log_record = SdkLogRecord( @@ -1674,9 +1749,7 @@ class OpenTelemetry(CustomLogger): ########## LLM Request Medssages / tools / content Attributes ########### ######################################################################### - if litellm.turn_off_message_logging is True: - return - if self.message_logging is not True: + if not self._capture_in_span(): return if optional_params.get("tools"): diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 962806c5b52..97e57240a96 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -442,6 +442,103 @@ class TestOpenTelemetryDualHandlerIsolation(unittest.TestCase): ) +class TestOpenTelemetryCaptureMessageContent(unittest.TestCase): + """OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT and the + OpenTelemetryConfig.capture_message_content programmatic override + drive what the handler captures in spans vs events.""" + + @staticmethod + def _make(env=None, config_value=None, message_logging=True): + env_dict = ( + {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": env} + if env is not None + else {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": ""} + ) + with patch.dict(os.environ, env_dict): + handler = OpenTelemetry( + config=OpenTelemetryConfig( + exporter="console", capture_message_content=config_value + ) + ) + handler.message_logging = message_logging + return handler, handler._resolve_capture_mode() + + def test_no_explicit_setting_falls_back_to_message_logging_true(self): + _, mode = self._make() + self.assertEqual(mode, "SPAN_AND_EVENT") + + def test_no_explicit_setting_falls_back_to_message_logging_false(self): + _, mode = self._make(message_logging=False) + self.assertEqual(mode, "NO_CONTENT") + + def test_env_var_no_content(self): + _, mode = self._make(env="NO_CONTENT") + self.assertEqual(mode, "NO_CONTENT") + + def test_env_var_span_only(self): + _, mode = self._make(env="SPAN_ONLY") + self.assertEqual(mode, "SPAN_ONLY") + + def test_env_var_event_only(self): + _, mode = self._make(env="EVENT_ONLY") + self.assertEqual(mode, "EVENT_ONLY") + + def test_env_var_span_and_event(self): + _, mode = self._make(env="SPAN_AND_EVENT") + self.assertEqual(mode, "SPAN_AND_EVENT") + + def test_env_var_legacy_true_maps_to_event_only(self): + _, mode = self._make(env="true") + self.assertEqual(mode, "EVENT_ONLY") + + def test_env_var_unknown_value_falls_through_to_legacy(self): + _, mode = self._make(env="garbage", message_logging=True) + self.assertEqual(mode, "SPAN_AND_EVENT") + + def test_config_field_overrides_env(self): + _, mode = self._make(env="EVENT_ONLY", config_value="SPAN_ONLY") + self.assertEqual(mode, "SPAN_ONLY") + + def test_turn_off_message_logging_forces_no_content(self): + with patch("litellm.turn_off_message_logging", True): + _, mode = self._make(env="SPAN_AND_EVENT", message_logging=True) + self.assertEqual(mode, "NO_CONTENT") + + def test_capture_in_span_and_event_predicates(self): + cases = { + "NO_CONTENT": (False, False), + "SPAN_ONLY": (True, False), + "EVENT_ONLY": (False, True), + "SPAN_AND_EVENT": (True, True), + } + for mode, (in_span, in_event) in cases.items(): + handler, _ = self._make(env=mode) + self.assertEqual(handler._capture_in_span(), in_span, msg=mode) + self.assertEqual(handler._capture_in_event(), in_event, msg=mode) + + def test_two_handlers_can_have_different_modes(self): + # FIL's stated requirement: one handler strips content, the other keeps it. + with patch.dict( + os.environ, {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": ""} + ): + stripped = OpenTelemetry( + config=OpenTelemetryConfig( + exporter="console", capture_message_content="NO_CONTENT" + ) + ) + kept = OpenTelemetry( + config=OpenTelemetryConfig( + exporter="console", capture_message_content="SPAN_AND_EVENT" + ) + ) + self.assertEqual(stripped._resolve_capture_mode(), "NO_CONTENT") + self.assertEqual(kept._resolve_capture_mode(), "SPAN_AND_EVENT") + self.assertFalse(stripped._capture_in_span()) + self.assertFalse(stripped._capture_in_event()) + self.assertTrue(kept._capture_in_span()) + self.assertTrue(kept._capture_in_event()) + + class TestOpenTelemetry(unittest.TestCase): POLL_INTERVAL = 0.05 POLL_TIMEOUT = 2.0 @@ -1067,6 +1164,7 @@ class TestOpenTelemetry(unittest.TestCase): result = otel._get_span_name(kwargs) self.assertEqual(result, LITELLM_REQUEST_SPAN_NAME) + @patch.dict(os.environ, {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": ""}) @patch("litellm.turn_off_message_logging", False) def test_maybe_log_raw_request_creates_span(self): """Test _maybe_log_raw_request creates span when logging enabled""" @@ -2194,6 +2292,19 @@ class TestOpenTelemetrySemanticConventions138(unittest.TestCase): See: https://github.com/BerriAI/litellm/issues/17794 """ + def setUp(self): + # Insulate from a shell-set OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT + # so these tests exercise the legacy default path (message_logging=True). + self._prev = os.environ.pop( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", None + ) + + def tearDown(self): + if self._prev is not None: + os.environ["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"] = ( + self._prev + ) + def test_input_messages_uses_parts_structure(self): """ Test that gen_ai.input.messages uses the OTEL 1.38 parts array structure. From 492118de2515a2e2b327013c7004080e3d36d87d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 7 May 2026 18:12:23 +0000 Subject: [PATCH 24/85] Fix OTEL false content capture mode --- litellm/integrations/opentelemetry.py | 3 +++ tests/test_litellm/integrations/test_opentelemetry.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index e49f0fabdaa..0af016feb26 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -329,6 +329,7 @@ class OpenTelemetry(CustomLogger): ``self.message_logging`` flag is consulted dynamically per request). ``"true"``/``"1"`` map to ``EVENT_ONLY`` per the contrib convention. + ``"false"``/``"0"`` map to ``NO_CONTENT``. Unknown values are ignored. """ explicit = self.config.capture_message_content or os.getenv( @@ -339,6 +340,8 @@ class OpenTelemetry(CustomLogger): normalized = explicit.upper() if normalized in ("TRUE", "1"): return CAPTURE_MODE_EVENT_ONLY + if normalized in ("FALSE", "0"): + return CAPTURE_MODE_NO_CONTENT if normalized in _VALID_CAPTURE_MODES: return normalized return None diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 97e57240a96..bea718dfea0 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -491,6 +491,12 @@ class TestOpenTelemetryCaptureMessageContent(unittest.TestCase): _, mode = self._make(env="true") self.assertEqual(mode, "EVENT_ONLY") + def test_env_var_legacy_false_maps_to_no_content(self): + for env in ("false", "0"): + with self.subTest(env=env): + _, mode = self._make(env=env) + self.assertEqual(mode, "NO_CONTENT") + def test_env_var_unknown_value_falls_through_to_legacy(self): _, mode = self._make(env="garbage", message_logging=True) self.assertEqual(mode, "SPAN_AND_EVENT") From 158b0c28c02556a80bf5992ad750aacb9d40ba6f Mon Sep 17 00:00:00 2001 From: "oss-pr-review-agent-shin[bot]" <281797381+oss-pr-review-agent-shin[bot]@users.noreply.github.com> Date: Thu, 7 May 2026 21:29:47 +0000 Subject: [PATCH 25/85] =?UTF-8?q?[litellm-agent]=20Staging=20=E2=86=92=20l?= =?UTF-8?q?itellm=5Finternal=5Fstaging=20(5/7/2026)=20(#27375)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squash-merged by litellm-agent from oss-pr-review-agent-shin[bot]'s PR. --- .../mcp_server/discoverable_endpoints.py | 79 ++---- .../_experimental/mcp_server/oauth_utils.py | 109 ++++++++- litellm/proxy/auth/handle_jwt.py | 30 +++ .../proxy/guardrails/guardrail_endpoints.py | 20 +- .../proxy/guardrails/guardrail_registry.py | 65 ++++- litellm/proxy/guardrails/init_guardrails.py | 1 + .../health_endpoints/_health_endpoints.py | 67 +++-- .../tag_management_endpoints.py | 51 +++- litellm/proxy/management_endpoints/ui_sso.py | 4 +- litellm/proxy/proxy_server.py | 10 + .../mcp_server/test_byok_oauth_endpoints.py | 66 +++++ .../mcp_server/test_discoverable_endpoints.py | 69 +++++- .../proxy/auth/test_handle_jwt.py | 71 +++++- .../guardrails/test_guardrail_endpoints.py | 64 +++++ .../guardrails/test_guardrail_registry.py | 120 +++++++++ .../health_endpoints/test_health_endpoints.py | 230 ++++++++++++++++++ .../test_tag_management_endpoints.py | 111 +++++++++ .../UsagePage/components/UsagePageView.tsx | 41 ++-- .../src/components/model_info_view.test.tsx | 27 ++ .../src/components/model_info_view.tsx | 6 + .../src/components/networking.tsx | 21 +- .../components/templates/key_info_view.tsx | 4 +- 22 files changed, 1145 insertions(+), 121 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 1794cd14381..62691641234 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -12,7 +12,8 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.proxy._experimental.mcp_server.oauth_utils import ( TOKEN_NO_CACHE_HEADERS, - validate_loopback_redirect_uri, + get_request_base_url, + validate_trusted_redirect_uri, ) from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.common_utils.encrypt_decrypt_utils import ( @@ -29,51 +30,6 @@ router = APIRouter( ) -def get_request_base_url(request: Request) -> str: - """ - Get the base URL for the request, considering X-Forwarded-* headers. - - X-Forwarded-Proto / X-Forwarded-Host / X-Forwarded-Port are only honoured - when the request comes from a configured trusted proxy - (``use_x_forwarded_for`` enabled AND caller in ``mcp_trusted_proxy_ranges``). - Otherwise the request's literal ``base_url`` is returned, so an - untrusted caller cannot poison OAuth-discovery / redirect_uri values - by injecting headers. - - Args: - request: FastAPI Request object - - Returns: - The reconstructed base URL (e.g., "https://proxy.example.com") - """ - base_url = str(request.base_url).rstrip("/") - parsed = urlparse(base_url) - - if not IPAddressUtils.is_request_from_trusted_proxy(request): - return base_url - - x_forwarded_proto = request.headers.get("X-Forwarded-Proto") - x_forwarded_host = request.headers.get("X-Forwarded-Host") - x_forwarded_port = request.headers.get("X-Forwarded-Port") - - scheme = x_forwarded_proto if x_forwarded_proto else parsed.scheme - - if x_forwarded_host: - # X-Forwarded-Host may already include port (e.g., "example.com:8080") - if ":" in x_forwarded_host and not x_forwarded_host.startswith("["): - netloc = x_forwarded_host - elif x_forwarded_port: - netloc = f"{x_forwarded_host}:{x_forwarded_port}" - else: - netloc = x_forwarded_host - else: - netloc = parsed.netloc - if x_forwarded_port and ":" not in netloc: - netloc = f"{netloc}:{x_forwarded_port}" - - return urlunparse((scheme, netloc, parsed.path, "", "", "")) - - def encode_state_with_base_url( base_url: str, original_state: str, @@ -127,12 +83,14 @@ def decode_state_hash(encrypted_state: str) -> dict: return state_data -def _get_validated_client_redirect_uri(state_data: Dict[str, Any]) -> str: - """Return a loopback client redirect URI from OAuth state.""" +def _get_validated_client_redirect_uri( + request: Request, state_data: Dict[str, Any] +) -> str: + """Return a trusted (same-origin or loopback) client redirect URI from OAuth state.""" redirect_uri = state_data.get("client_redirect_uri") or state_data.get("base_url") if not redirect_uri or not isinstance(redirect_uri, str): raise HTTPException(status_code=400, detail="Invalid redirect URI") - validate_loopback_redirect_uri(redirect_uri) + validate_trusted_redirect_uri(request, redirect_uri) return redirect_uri @@ -338,12 +296,12 @@ async def authorize_with_server( status_code=400, detail="MCP server authorization url is not set" ) - # Loopback-only redirect_uri. The URI is encrypted into the OAuth - # state and decoded on /callback to redirect the user back; a non- - # loopback URI would be an open-redirect + code-theft primitive - # (VERIA-57 root cause B). MCP clients are native apps — loopback is - # the spec-compliant callback pattern. - validate_loopback_redirect_uri(redirect_uri) + # Loopback OR same-origin redirect_uri. The URI is encrypted into the + # OAuth state and decoded on /callback to redirect the user back; + # restricting to trusted origins blocks the open-redirect + + # code-theft primitive (VERIA-57 root cause B). Loopback supports + # native MCP clients; same-origin supports the proxy's own UI callback. + validate_trusted_redirect_uri(request, redirect_uri) parsed = urlparse(redirect_uri) base_url = urlunparse(parsed._replace(query="")) request_base_url = get_request_base_url(request) @@ -660,17 +618,18 @@ async def token_endpoint( @router.get("/callback") -async def callback(code: str, state: str): +async def callback(request: Request, code: str, state: str): try: state_data = decode_state_hash(state) original_state = state_data["original_state"] - # Re-validate loopback at the sink. /authorize rejects non-loopback + # Re-validate at the sink. /authorize rejects untrusted # redirect_uri before encoding into state, but encrypted states # minted before that check was added have no expiry and remain - # valid indefinitely. Validating here blocks the open-redirect + - # code-theft primitive even for pre-fix states. - redirect_uri = _get_validated_client_redirect_uri(state_data) + # valid indefinitely. Validating here (same-origin OR loopback) + # blocks the open-redirect + code-theft primitive even for pre-fix + # states while allowing the UI's same-origin callback to work. + redirect_uri = _get_validated_client_redirect_uri(request, state_data) params = {"code": code, "state": original_state} complete_returned_url = _append_query_params(redirect_uri, params) diff --git a/litellm/proxy/_experimental/mcp_server/oauth_utils.py b/litellm/proxy/_experimental/mcp_server/oauth_utils.py index b13cf83058c..343d1bee613 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_utils.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_utils.py @@ -2,15 +2,63 @@ (BYOK + discoverable / pass-through OAuth proxy).""" from ipaddress import ip_address -from urllib.parse import urlparse +from urllib.parse import urlparse, urlunparse -from fastapi import HTTPException +from fastapi import HTTPException, Request + +from litellm._logging import verbose_logger +from litellm.proxy.auth.ip_address_utils import IPAddressUtils # RFC 6749 §5.1 / OAuth 2.1 draft-15 §4.1.3: token-endpoint responses # must not be cached — both success and error bodies may reveal secrets. TOKEN_NO_CACHE_HEADERS = {"Cache-Control": "no-store", "Pragma": "no-cache"} +def get_request_base_url(request: Request) -> str: + """ + Get the base URL for the request, considering X-Forwarded-* headers. + + X-Forwarded-Proto / X-Forwarded-Host / X-Forwarded-Port are only honoured + when the request comes from a configured trusted proxy + (``use_x_forwarded_for`` enabled AND caller in ``mcp_trusted_proxy_ranges``). + Otherwise the request's literal ``base_url`` is returned, so an + untrusted caller cannot poison OAuth-discovery / redirect_uri values + by injecting headers. + + Args: + request: FastAPI Request object + + Returns: + The reconstructed base URL (e.g., "https://proxy.example.com") + """ + base_url = str(request.base_url).rstrip("/") + parsed = urlparse(base_url) + + if not IPAddressUtils.is_request_from_trusted_proxy(request): + return base_url + + x_forwarded_proto = request.headers.get("X-Forwarded-Proto") + x_forwarded_host = request.headers.get("X-Forwarded-Host") + x_forwarded_port = request.headers.get("X-Forwarded-Port") + + scheme = x_forwarded_proto if x_forwarded_proto else parsed.scheme + + if x_forwarded_host: + # X-Forwarded-Host may already include port (e.g., "example.com:8080") + if ":" in x_forwarded_host and not x_forwarded_host.startswith("["): + netloc = x_forwarded_host + elif x_forwarded_port: + netloc = f"{x_forwarded_host}:{x_forwarded_port}" + else: + netloc = x_forwarded_host + else: + netloc = parsed.netloc + if x_forwarded_port and ":" not in netloc: + netloc = f"{netloc}:{x_forwarded_port}" + + return urlunparse((scheme, netloc, parsed.path, "", "", "")) + + def validate_loopback_redirect_uri(redirect_uri: str) -> None: """Require a loopback ``redirect_uri`` (OAuth 2.1 §4.1.2.1 + RFC 8252 §7.3 native-app pattern). MCP clients are native apps that listen on @@ -46,3 +94,60 @@ def validate_loopback_redirect_uri(redirect_uri: str) -> None: # don't let it bubble up as a 500. pass raise HTTPException(status_code=400, detail="invalid_request") + + +def validate_trusted_redirect_uri(request: Request, redirect_uri: str) -> None: + """Accept same-origin (proxy's own origin) OR loopback ``redirect_uri``. + + Same-origin is required for the LiteLLM UI's OAuth flow: the UI + redirects to ``/ui/mcp/oauth/callback`` which is not loopback + but is on the proxy's own trusted HTTPS origin. An attacker cannot + host content on the proxy's own origin without already owning the + proxy, so the open-redirect / code-theft primitive that motivated + :func:`validate_loopback_redirect_uri` does not apply here. + + Loopback continues to be accepted for native MCP clients (per + OAuth 2.1 §4.1.2.1 + RFC 8252 §7.3). + + Use this in the discoverable OAuth proxy endpoints that serve both + native clients and the proxy's own UI. BYOK endpoints that only + support native clients should keep + :func:`validate_loopback_redirect_uri`. + """ + try: + parsed = urlparse(redirect_uri) + except ValueError: + raise HTTPException(status_code=400, detail="invalid_request") + if parsed.scheme not in ("http", "https"): + raise HTTPException(status_code=400, detail="invalid_request") + if parsed.fragment: + raise HTTPException(status_code=400, detail="invalid_request") + + # Same-origin: scheme + netloc (host[:port]) must match the proxy's + # own base URL at this request (honouring trusted X-Forwarded-*). + try: + proxy_base = urlparse(get_request_base_url(request)) + if ( + parsed.netloc + and parsed.scheme == proxy_base.scheme + and parsed.netloc.lower() == proxy_base.netloc.lower() + ): + return + except Exception as exc: + # If we can't determine the proxy's origin, fall through to + # loopback. Log so the failure is diagnosable in production. + verbose_logger.warning( + "validate_trusted_redirect_uri: could not determine proxy origin, " + "falling back to loopback-only check. error=%s", + exc, + ) + + host = (parsed.hostname or "").lower() + if host == "localhost": + return + try: + if ip_address(host).is_loopback: + return + except ValueError: + pass + raise HTTPException(status_code=400, detail="invalid_request") diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index d1fd5818f35..a270031c33c 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -224,6 +224,36 @@ class JWTHandler: return [] + def get_all_jwt_team_ids(self, token: dict) -> List[str]: + """ + Return team IDs from both the plural ``team_ids_jwt_field`` and the + singular ``team_id_jwt_field`` claim, as a deduplicated list preserving + plural-first order. + + Membership-reconciliation paths (SSO callback, JWT-bearer sync) need + to consider both claim shapes. Reading only the plural field — as + callers historically did — silently dropped users whose IdP populates + the singular field, which is what Okta and Auth0 default to when a + user has a single primary team. + + This intentionally does NOT consult ``team_id_default``: that fallback + is a property of how the JWT-bearer auth flow resolves a single + request-bound team, not of the token's claims. Callers that want the + default-team behavior should still go through ``get_team_id``. + """ + team_ids: List[str] = list(self.get_team_ids_from_jwt(token)) + if self.litellm_jwtauth.team_id_jwt_field is not None: + singular = get_nested_value( + data=token, + key_path=self.litellm_jwtauth.team_id_jwt_field, + default=None, + ) + if isinstance(singular, list): + singular = singular[0] if singular else None + if singular and singular not in team_ids: + team_ids.append(singular) + return team_ids + def get_end_user_id( self, token: dict, default_value: Optional[str] ) -> Optional[str]: diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 5351391e5e1..e55f3b6e16b 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -242,6 +242,10 @@ async def list_guardrails_v2( gid = guardrail.get("guardrail_id") if gid in seen_guardrail_ids: continue + # Skip stale DB-backed entries — the DB row was deleted (likely by + # another pod) and reconciliation hasn't fired yet on this pod. + if gid is not None and IN_MEMORY_GUARDRAIL_HANDLER.get_source(gid) == "db": + continue if not is_admin: g_team_id = guardrail.get("team_id") if g_team_id is not None and g_team_id not in caller_team_ids: @@ -360,7 +364,7 @@ async def create_guardrail( try: IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail( - guardrail=cast(Guardrail, result) + guardrail=cast(Guardrail, result), source="db" ) verbose_proxy_logger.info( f"Immediate sync: Successfully initialized guardrail '{guardrail_name}' (ID: {guardrail_id})" @@ -1017,7 +1021,7 @@ async def approve_guardrail_submission( } try: IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail( - guardrail=cast(Guardrail, guardrail_dict) + guardrail=cast(Guardrail, guardrail_dict), source="db" ) verbose_proxy_logger.info( "Approved guardrail %s (ID: %s) and initialized in memory", @@ -1295,10 +1299,18 @@ async def get_guardrail_info(guardrail_id: str): guardrail_id=guardrail_id, prisma_client=prisma_client ) if result is None: - result = IN_MEMORY_GUARDRAIL_HANDLER.get_guardrail_by_id( + in_memory = IN_MEMORY_GUARDRAIL_HANDLER.get_guardrail_by_id( guardrail_id=guardrail_id ) - guardrail_definition_location = GUARDRAIL_DEFINITION_LOCATION.CONFIG + # Only return config-loaded entries here. A DB-backed entry that's + # missing from the DB is stale (deleted on another pod, awaiting + # reconciliation on this one) and must surface as 404. + if ( + in_memory is not None + and IN_MEMORY_GUARDRAIL_HANDLER.get_source(guardrail_id) == "config" + ): + result = in_memory + guardrail_definition_location = GUARDRAIL_DEFINITION_LOCATION.CONFIG if result is None: raise HTTPException( diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 868b23756d2..3aa612c761e 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -3,7 +3,7 @@ import importlib import os from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Type, cast +from typing import Any, Dict, List, Literal, Optional, Set, Type, cast import litellm from litellm import Router @@ -403,11 +403,19 @@ class InMemoryGuardrailHandler: Guardrail id to CustomGuardrail object mapping """ + self._sources: Dict[str, Literal["db", "config"]] = {} + """ + Guardrail id to provenance marker. "db" entries are reconciled against + the DB on each polling tick; "config" entries are owned by proxy_config.yaml + and never deleted by reconciliation. + """ + def initialize_guardrail( self, guardrail: Guardrail, config_file_path: Optional[str] = None, llm_router: Optional["Router"] = None, + source: Literal["db", "config"] = "config", ) -> Optional[Guardrail]: """ Initialize a guardrail from a dictionary and add it to the litellm callback manager @@ -420,6 +428,10 @@ class InMemoryGuardrailHandler: verbose_proxy_logger.debug( "guardrail_id already exists in IN_MEMORY_GUARDRAILS" ) + # Honor the caller's source even on the early-return path so a + # racing polling tick or a hot-reload of config can correct an + # entry's provenance. + self._sources[guardrail_id] = source return self.IN_MEMORY_GUARDRAILS[guardrail_id] custom_guardrail_callback: Optional[CustomGuardrail] = None @@ -492,6 +504,7 @@ class InMemoryGuardrailHandler: # store references to the guardrail in memory self.IN_MEMORY_GUARDRAILS[guardrail_id] = parsed_guardrail self.guardrail_id_to_custom_guardrail[guardrail_id] = custom_guardrail_callback + self._sources[guardrail_id] = source return parsed_guardrail @@ -552,7 +565,10 @@ class InMemoryGuardrailHandler: return _guardrail_callback def update_in_memory_guardrail( - self, guardrail_id: str, guardrail: Guardrail + self, + guardrail_id: str, + guardrail: Guardrail, + source: Literal["db", "config"] = "db", ) -> None: """ Update a guardrail in memory @@ -561,6 +577,7 @@ class InMemoryGuardrailHandler: - updates the guardrail params in litellm.callback_manager """ self.IN_MEMORY_GUARDRAILS[guardrail_id] = guardrail + self._sources[guardrail_id] = source custom_guardrail_callback = self.guardrail_id_to_custom_guardrail.get( guardrail_id @@ -579,6 +596,7 @@ class InMemoryGuardrailHandler: """ # Remove from in-memory storage self.IN_MEMORY_GUARDRAILS.pop(guardrail_id, None) + self._sources.pop(guardrail_id, None) # Remove the callback from litellm.callbacks custom_guardrail_callback = self.guardrail_id_to_custom_guardrail.pop( @@ -603,6 +621,34 @@ class InMemoryGuardrailHandler: """ return self.IN_MEMORY_GUARDRAILS.get(guardrail_id) + def get_source(self, guardrail_id: str) -> Optional[Literal["db", "config"]]: + """ + Return the provenance of an in-memory guardrail. + """ + return self._sources.get(guardrail_id) + + def reconcile_db_guardrails(self, db_guardrail_ids: Set[str]) -> List[str]: + """ + Drop in-memory entries that originated from the DB but are no longer + present in db_guardrail_ids. Config-loaded guardrails are never touched. + + Called by the periodic DB polling tick so that a guardrail deleted + on another pod is eventually purged from this pod's memory + callbacks. + """ + stale_ids = [ + guardrail_id + for guardrail_id, source in self._sources.items() + if source == "db" and guardrail_id not in db_guardrail_ids + ] + for guardrail_id in stale_ids: + verbose_proxy_logger.info( + "Reconcile: removing stale DB-backed guardrail '%s' from memory " + "(deleted in DB by another pod)", + guardrail_id, + ) + self.delete_in_memory_guardrail(guardrail_id) + return stale_ids + def _has_guardrail_params_changed( self, guardrail_id: str, new_guardrail: Guardrail ) -> bool: @@ -656,7 +702,10 @@ class InMemoryGuardrailHandler: return len(changed_fields) > 0 def reinitialize_guardrail( - self, guardrail: Guardrail, config_file_path: Optional[str] = None + self, + guardrail: Guardrail, + config_file_path: Optional[str] = None, + source: Literal["db", "config"] = "config", ) -> Optional[Guardrail]: """ Force re-initialization of a guardrail even if it exists in memory. @@ -675,7 +724,7 @@ class InMemoryGuardrailHandler: # Initialize fresh (will add new callback to litellm.callbacks) return self.initialize_guardrail( - guardrail=guardrail, config_file_path=config_file_path + guardrail=guardrail, config_file_path=config_file_path, source=source ) def sync_guardrail_from_db( @@ -696,9 +745,15 @@ class InMemoryGuardrailHandler: f"Guardrail '{guardrail_name}' (ID: {guardrail_id}) params changed, re-initializing..." ) return self.reinitialize_guardrail( - guardrail=guardrail, config_file_path=config_file_path + guardrail=guardrail, + config_file_path=config_file_path, + source="db", ) + # Params unchanged but the entry is still DB-backed; make sure the + # source marker reflects that even if it was previously set differently + # (e.g. a config entry whose UUID later collided with a DB row). + self._sources[guardrail_id] = "db" return self.IN_MEMORY_GUARDRAILS.get(guardrail_id) diff --git a/litellm/proxy/guardrails/init_guardrails.py b/litellm/proxy/guardrails/init_guardrails.py index d742cc223b4..83f1281dc02 100644 --- a/litellm/proxy/guardrails/init_guardrails.py +++ b/litellm/proxy/guardrails/init_guardrails.py @@ -30,6 +30,7 @@ def init_guardrails_v2( guardrail=cast(Guardrail, guardrail), config_file_path=config_file_path, llm_router=llm_router, + source="config", ) if initialized_guardrail: guardrail_list.append(initialized_guardrail) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 096e23e673d..2c857c0fde8 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -1742,29 +1742,54 @@ async def test_model_connection( # Look up model configuration from router if model name is provided # This gets the litellm_params from proxy config (with resolved env vars) config_litellm_params: dict = {} - if model_name and llm_router is not None: + if llm_router is not None: + # Prefer disambiguation by deployment id (`model_info.id`) when + # the caller supplies it. This is required when multiple + # deployments share a `model_name` (e.g. wildcard `openai/*` + # with multiple `api_base` values for failover): the UI's + # "Test Connection" button targets a specific row, and that + # row's id is the only thing that uniquely identifies which + # deployment to probe. Without this, all duplicates collapse + # onto `deployments[0]`. + request_model_info = model_info or {} + request_model_id = request_model_info.get("id") try: - # First try to find by proxy model_name (e.g., "gpt-4o") - deployments = llm_router.get_model_list(model_name=model_name) - - # If not found, try to find by litellm model name (e.g., "azure/gpt-4o") - if not deployments or len(deployments) == 0: - all_deployments = llm_router.get_model_list(model_name=None) - if all_deployments: - for deployment in all_deployments: - if ( - deployment.get("litellm_params", {}).get("model") - == model_name - ): - deployments = [deployment] - break - - if deployments and len(deployments) > 0: - # Use the first deployment's litellm_params as base config - # These already have resolved environment variables from proxy config - config_litellm_params = dict( - deployments[0].get("litellm_params", {}) + deployment_by_id = None + if request_model_id: + deployment_by_id = llm_router.get_deployment( + model_id=request_model_id ) + + if deployment_by_id is not None: + config_litellm_params = deployment_by_id.litellm_params.model_dump( + exclude_none=True + ) + elif model_name: + # Fall back to model_name lookup for callers (e.g. the + # "Add Model" wizard, or curl) that don't supply an id. + # First try to find by proxy model_name (e.g., "gpt-4o") + deployments = llm_router.get_model_list(model_name=model_name) + + # If not found, try to find by litellm model name + # (e.g., "azure/gpt-4o") + if not deployments or len(deployments) == 0: + all_deployments = llm_router.get_model_list(model_name=None) + if all_deployments: + for deployment in all_deployments: + if ( + deployment.get("litellm_params", {}).get("model") + == model_name + ): + deployments = [deployment] + break + + if deployments and len(deployments) > 0: + # Use the first deployment's litellm_params as base + # config. These already have resolved environment + # variables from proxy config. + config_litellm_params = dict( + deployments[0].get("litellm_params", {}) + ) except Exception as e: verbose_proxy_logger.debug( f"Could not find model {model_name} in router: {e}. " diff --git a/litellm/proxy/management_endpoints/tag_management_endpoints.py b/litellm/proxy/management_endpoints/tag_management_endpoints.py index 0e60820aab1..2a4895d0299 100644 --- a/litellm/proxy/management_endpoints/tag_management_endpoints.py +++ b/litellm/proxy/management_endpoints/tag_management_endpoints.py @@ -12,9 +12,10 @@ All /tag management endpoints import asyncio import json -from typing import TYPE_CHECKING, Dict, List, Optional +from datetime import datetime +from typing import TYPE_CHECKING, Any, Dict, List, Optional -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query from litellm._logging import verbose_proxy_logger from litellm.proxy._types import UserAPIKeyAuth @@ -395,6 +396,32 @@ async def info_tag( raise HTTPException(status_code=500, detail=str(e)) +def _validate_tag_list_date_range( + start_date: Optional[str], end_date: Optional[str] +) -> None: + """Require both dates together, and enforce YYYY-MM-DD format with start <= end.""" + if (start_date is None) != (end_date is None): + raise HTTPException( + status_code=400, + detail="start_date and end_date must be provided together", + ) + if start_date is None: + return + try: + start = datetime.strptime(start_date, "%Y-%m-%d") + end = datetime.strptime(end_date, "%Y-%m-%d") # type: ignore[arg-type] + except ValueError as e: + raise HTTPException( + status_code=400, + detail=f"Invalid date format, expected YYYY-MM-DD: {e}", + ) + if start > end: + raise HTTPException( + status_code=400, + detail="start_date must be on or before end_date", + ) + + @router.get( "/tag/list", tags=["tag management"], @@ -402,6 +429,18 @@ async def info_tag( ) async def list_tags( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + start_date: Optional[str] = Query( + None, + description=( + "Optional start date (YYYY-MM-DD). When provided together with " + "end_date, dynamic tags are limited to those active in the window. " + "Stored tags are always returned." + ), + ), + end_date: Optional[str] = Query( + None, + description="Optional end date (YYYY-MM-DD). Must be given with start_date.", + ), ): """ List all available tags with their budget information. @@ -411,6 +450,8 @@ async def list_tags( if prisma_client is None: raise HTTPException(status_code=500, detail="Database not connected") + _validate_tag_list_date_range(start_date, end_date) + try: ## QUERY STORED TAGS ## tag_records = await prisma_client.db.litellm_tagtable.find_many( @@ -453,9 +494,13 @@ async def list_tags( # Prisma's distinct fetches all columns for all rows and deduplicates # in application code, which is extremely slow on large tables. # See: https://www.prisma.io/docs/orm/prisma-client/queries/aggregation-grouping-summarizing#distinct-under-the-hood + dynamic_tag_where: Dict[str, Any] = {"tag": {"not": None}} + if start_date is not None and end_date is not None: + dynamic_tag_where["date"] = {"gte": start_date, "lte": end_date} + dynamic_tag_rows = await prisma_client.db.litellm_dailytagspend.group_by( by=["tag"], - where={"tag": {"not": None}}, + where=dynamic_tag_where, min={"created_at": True}, max={"updated_at": True}, ) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 74ee7c7220d..ea629f24485 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -740,7 +740,7 @@ def generic_response_convertor( all_teams = [] if sso_jwt_handler is not None: - team_ids = sso_jwt_handler.get_team_ids_from_jwt(cast(dict, response)) + team_ids = sso_jwt_handler.get_all_jwt_team_ids(cast(dict, response)) all_teams.extend(team_ids) if team_mappings is not None and team_mappings.team_ids_jwt_field is not None: @@ -755,7 +755,7 @@ def generic_response_convertor( f"Loaded team_ids from DB team_mappings.team_ids_jwt_field='{team_mappings.team_ids_jwt_field}': {team_ids_from_db_mapping}" ) else: - team_ids = jwt_handler.get_team_ids_from_jwt(cast(dict, response)) + team_ids = jwt_handler.get_all_jwt_team_ids(cast(dict, response)) all_teams.extend(team_ids) # Determine user role based on role_mappings if available diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c96d0acb008..f0b3faa7267 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5936,10 +5936,20 @@ class ProxyConfig: verbose_proxy_logger.debug( "guardrails from the DB %s", str(guardrails_in_db) ) + db_guardrail_ids: set = set() for guardrail in guardrails_in_db: + guardrail_id = guardrail.get("guardrail_id") + if guardrail_id: + db_guardrail_ids.add(guardrail_id) IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db( guardrail=cast(Guardrail, guardrail), ) + + # Drop in-memory DB-backed entries whose row was deleted on another + # pod. Config-loaded entries are never touched. + IN_MEMORY_GUARDRAIL_HANDLER.reconcile_db_guardrails( + db_guardrail_ids=db_guardrail_ids + ) except Exception as e: verbose_proxy_logger.exception( "litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - {}".format( diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py index 89992c510f5..9f004318488 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py @@ -1135,3 +1135,69 @@ def test_validate_loopback_redirect_uri_rejects_malformed_cleanly(): with pytest.raises(HTTPException) as exc: validate_loopback_redirect_uri("http://[not-an-ip]/cb") assert exc.value.status_code == 400 + + +def _mock_request_with_base_url(base_url: str): + req = MagicMock() + req.base_url = base_url + req.headers = {} + return req + + +def test_validate_trusted_redirect_uri_accepts_same_origin(): + """UI OAuth flow: redirect_uri on the proxy's own origin is allowed.""" + from litellm.proxy._experimental.mcp_server.oauth_utils import ( + validate_trusted_redirect_uri, + ) + + req = _mock_request_with_base_url("https://proxy.example.com/") + # Should not raise. + validate_trusted_redirect_uri( + req, "https://proxy.example.com/ui/mcp/oauth/callback" + ) + + +def test_validate_trusted_redirect_uri_accepts_loopback(): + """Native MCP client flow: loopback is still allowed.""" + from litellm.proxy._experimental.mcp_server.oauth_utils import ( + validate_trusted_redirect_uri, + ) + + req = _mock_request_with_base_url("https://proxy.example.com/") + validate_trusted_redirect_uri(req, "http://127.0.0.1:3000/cb") + validate_trusted_redirect_uri(req, "http://localhost:3000/cb") + + +def test_validate_trusted_redirect_uri_rejects_external_origin(): + """An attacker-controlled origin must still be rejected.""" + from litellm.proxy._experimental.mcp_server.oauth_utils import ( + validate_trusted_redirect_uri, + ) + + req = _mock_request_with_base_url("https://proxy.example.com/") + with pytest.raises(HTTPException) as exc: + validate_trusted_redirect_uri(req, "https://attacker.example.com/cb") + assert exc.value.status_code == 400 + + +def test_validate_trusted_redirect_uri_rejects_scheme_mismatch(): + """https→http (or vice versa) on the same host is not same-origin.""" + from litellm.proxy._experimental.mcp_server.oauth_utils import ( + validate_trusted_redirect_uri, + ) + + req = _mock_request_with_base_url("https://proxy.example.com/") + with pytest.raises(HTTPException) as exc: + validate_trusted_redirect_uri(req, "http://proxy.example.com/ui/callback") + assert exc.value.status_code == 400 + + +def test_validate_trusted_redirect_uri_rejects_fragment(): + from litellm.proxy._experimental.mcp_server.oauth_utils import ( + validate_trusted_redirect_uri, + ) + + req = _mock_request_with_base_url("https://proxy.example.com/") + with pytest.raises(HTTPException) as exc: + validate_trusted_redirect_uri(req, "https://proxy.example.com/ui/cb#code=1") + assert exc.value.status_code == 400 diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 85d5d6ba466..581324d47d2 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -23,6 +23,20 @@ def mock_mcp_client_ip(): yield +def _mock_callback_request(base_url: str = "http://localhost:3000/"): + """Return a MagicMock Request for callback/authorize same-origin tests. + + The callback handler only uses ``request`` to compute the proxy's own + base URL via ``get_request_base_url`` (which reads ``request.base_url`` + and trusted ``X-Forwarded-*`` headers). A simple MagicMock with the + right attributes is sufficient. + """ + req = MagicMock() + req.base_url = base_url + req.headers = {} + return req + + @pytest.fixture def trust_xff(): """Force ``IPAddressUtils.is_request_from_trusted_proxy`` to True. @@ -1844,6 +1858,7 @@ async def test_oauth_callback_redirects_with_state(): # Call callback endpoint with code and state response = await callback( + request=_mock_callback_request(), code="test_authorization_code_12345", state="encrypted_state_value", ) @@ -1887,6 +1902,7 @@ async def test_oauth_callback_preserves_client_redirect_uri_query(): } response = await callback( + request=_mock_callback_request(), code="test_authorization_code_12345", state="encrypted_state_value", ) @@ -1917,6 +1933,7 @@ async def test_oauth_callback_handles_invalid_state(): # Call callback endpoint with invalid state response = await callback( + request=_mock_callback_request(), code="test_code", state="invalid_encrypted_state", ) @@ -1926,6 +1943,40 @@ async def test_oauth_callback_handles_invalid_state(): assert "Authentication incomplete" in response.body.decode() +@pytest.mark.asyncio +async def test_oauth_callback_accepts_same_origin_ui_redirect(): + """UI OAuth flow: the callback should redirect to the proxy's own UI + origin when the encrypted state carries a same-origin client_redirect_uri.""" + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + callback, + ) + + with patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.decode_state_hash" + ) as mock_decode: + mock_decode.return_value = { + "base_url": "https://proxy.example.com/ui/mcp/oauth/callback", + "original_state": "state-123", + "code_challenge": None, + "code_challenge_method": None, + "client_redirect_uri": "https://proxy.example.com/ui/mcp/oauth/callback", + } + + response = await callback( + request=_mock_callback_request(base_url="https://proxy.example.com/"), + code="auth-code-123", + state="encrypted_state", + ) + + assert response.status_code == 302 + assert ( + "https://proxy.example.com/ui/mcp/oauth/callback" + in response.headers["location"] + ) + assert "code=auth-code-123" in response.headers["location"] + assert "state=state-123" in response.headers["location"] + + @pytest.mark.asyncio async def test_oauth_authorize_includes_scopes_from_server_config(): """Test that authorize endpoint includes scopes from server configuration.""" @@ -2307,7 +2358,11 @@ async def test_callback_revalidates_loopback_on_decoded_base_url(): "client_redirect_uri": "https://attacker.example.com/cb", } with pytest.raises(HTTPException) as exc_info: - await callback(code="stolen_code", state="encrypted_stale_state") + await callback( + request=_mock_callback_request(), + code="stolen_code", + state="encrypted_stale_state", + ) assert exc_info.value.status_code == 400 @@ -2329,7 +2384,11 @@ async def test_callback_revalidates_loopback_on_decoded_client_redirect_uri(): "client_redirect_uri": "https://attacker.example.com/cb", } with pytest.raises(HTTPException) as exc_info: - await callback(code="stolen_code", state="encrypted_stale_state") + await callback( + request=_mock_callback_request(), + code="stolen_code", + state="encrypted_stale_state", + ) assert exc_info.value.status_code == 400 @@ -2349,7 +2408,11 @@ async def test_callback_rejects_state_missing_redirect_uri(): "code_challenge_method": None, } with pytest.raises(HTTPException) as exc_info: - await callback(code="code", state="encrypted_malformed_state") + await callback( + request=_mock_callback_request(), + code="code", + state="encrypted_malformed_state", + ) assert exc_info.value.status_code == 400 diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index b7dba9c1d16..c09c303deee 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -1,5 +1,5 @@ from typing import Optional -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -494,6 +494,75 @@ async def test_sync_user_role_and_teams_no_cache_write_when_nothing_changes(): mock_cache.async_set_cache.assert_not_called() +def test_get_all_jwt_team_ids_unions_singular_and_plural(): + """get_all_jwt_team_ids must include the singular team_id_jwt_field claim + in addition to the plural team_ids_jwt_field, deduplicated.""" + jwt_handler = JWTHandler() + jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=MagicMock(), + litellm_jwtauth=LiteLLM_JWTAuth( + team_id_jwt_field="team_id", + team_ids_jwt_field="teams", + ), + ) + + # singular only — Okta/Auth0 default shape + assert jwt_handler.get_all_jwt_team_ids({"team_id": "team-low"}) == ["team-low"] + + # plural only — pre-fix shape + assert jwt_handler.get_all_jwt_team_ids({"teams": ["a", "b"]}) == ["a", "b"] + + # both populated, no overlap + assert jwt_handler.get_all_jwt_team_ids( + {"team_id": "primary", "teams": ["a", "b"]} + ) == ["a", "b", "primary"] + + # both populated with overlap — singular dedup'd + assert jwt_handler.get_all_jwt_team_ids({"team_id": "a", "teams": ["a", "b"]}) == [ + "a", + "b", + ] + + # neither populated + assert jwt_handler.get_all_jwt_team_ids({}) == [] + + +def test_get_all_jwt_team_ids_does_not_use_team_id_default(): + """team_id_default is a JWT-bearer-flow auth-builder fallback, not a token + claim. It must NOT leak into get_all_jwt_team_ids — otherwise SSO logins + would silently start adding users to the default team for any tenant that + has team_id_default configured.""" + jwt_handler = JWTHandler() + jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=MagicMock(), + litellm_jwtauth=LiteLLM_JWTAuth( + team_id_jwt_field="team_id", + team_ids_jwt_field="teams", + team_id_default="default-team", + ), + ) + + # team_id claim missing — must not fall back to default-team + assert jwt_handler.get_all_jwt_team_ids({"teams": []}) == [] + assert jwt_handler.get_all_jwt_team_ids({}) == [] + + # only the plural is populated — default still must not be added + assert jwt_handler.get_all_jwt_team_ids({"teams": ["a"]}) == ["a"] + + # team_id_jwt_field unset entirely + only default configured: still no default + jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=MagicMock(), + litellm_jwtauth=LiteLLM_JWTAuth( + team_ids_jwt_field="teams", + team_id_default="default-team", + ), + ) + assert jwt_handler.get_all_jwt_team_ids({"teams": []}) == [] + + @pytest.mark.asyncio async def test_map_jwt_role_to_litellm_role(): """Test JWT role mapping to LiteLLM roles with various patterns""" diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py index 033deb3ff42..0d7becd3e2e 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py @@ -106,9 +106,11 @@ def mock_in_memory_handler(mocker): mock_handler = mocker.Mock(spec=InMemoryGuardrailHandler) mock_handler.list_in_memory_guardrails.return_value = [MOCK_CONFIG_GUARDRAIL] mock_handler.get_guardrail_by_id.return_value = MOCK_CONFIG_GUARDRAIL + mock_handler.get_source.return_value = "config" mock_handler.initialize_guardrail = mocker.Mock() mock_handler.update_in_memory_guardrail = mocker.Mock() mock_handler.delete_in_memory_guardrail = mocker.Mock() + mock_handler.reconcile_db_guardrails = mocker.Mock(return_value=[]) return mock_handler @@ -162,6 +164,67 @@ async def test_list_guardrails_v2_with_db_and_config( assert isinstance(config_guardrail.litellm_params, BaseLitellmParams) +@pytest.mark.asyncio +async def test_list_guardrails_v2_skips_stale_db_backed_in_memory_entries(mocker): + """ + A guardrail that's still in this pod's memory tagged source='db' but is no + longer in the DB result (deleted on another pod, awaiting reconcile) must + NOT surface in the list response — pre-fix it leaked as 'config'. + """ + stale_guardrail = { + "guardrail_id": "stale-db-id", + "guardrail_name": "Stale DB Guardrail", + "litellm_params": {"guardrail": "bedrock", "mode": "pre_call"}, + "guardrail_info": {}, + } + mock_prisma_client = mocker.Mock() + mock_prisma_client.db = mocker.Mock() + mock_prisma_client.db.litellm_guardrailstable = mocker.Mock() + mock_prisma_client.db.litellm_guardrailstable.find_many = AsyncMock(return_value=[]) + + mock_in_memory_handler = mocker.Mock() + mock_in_memory_handler.list_in_memory_guardrails.return_value = [stale_guardrail] + mock_in_memory_handler.get_source.return_value = "db" + + mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mocker.patch( + "litellm.proxy.guardrails.guardrail_registry.IN_MEMORY_GUARDRAIL_HANDLER", + mock_in_memory_handler, + ) + + admin_auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + response = await list_guardrails_v2(user_api_key_dict=admin_auth) + + assert response.guardrails == [] + mock_in_memory_handler.get_source.assert_called_with("stale-db-id") + + +@pytest.mark.asyncio +async def test_get_guardrail_info_404s_stale_db_backed_entry( + mocker, mock_prisma_client, mock_in_memory_handler +): + """ + Stale DB-backed entry (in-memory but not in DB) must 404 instead of being + returned as if it were a config-loaded guardrail. + """ + mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mocker.patch( + "litellm.proxy.guardrails.guardrail_registry.IN_MEMORY_GUARDRAIL_HANDLER", + mock_in_memory_handler, + ) + mock_prisma_client.db.litellm_guardrailstable.find_unique = AsyncMock( + return_value=None + ) + # In-memory still has it, but it's tagged as 'db' (stale, awaiting reconcile) + mock_in_memory_handler.get_source.return_value = "db" + + with pytest.raises(HTTPException) as exc_info: + await get_guardrail_info("stale-db-id") + + assert exc_info.value.status_code == 404 + assert "not found" in str(exc_info.value.detail) + + @pytest.mark.asyncio async def test_list_guardrails_v2_masks_sensitive_data_in_db_guardrails(mocker): """Test that sensitive litellm_params are masked for DB guardrails in list response""" @@ -1160,6 +1223,7 @@ async def test_get_guardrail_info_endpoint_config_guardrail(mocker): # Mock IN_MEMORY_GUARDRAIL_HANDLER at its source to return config guardrail mock_in_memory_handler = mocker.Mock() mock_in_memory_handler.get_guardrail_by_id.return_value = MOCK_CONFIG_GUARDRAIL + mock_in_memory_handler.get_source.return_value = "config" mocker.patch( "litellm.proxy.guardrails.guardrail_registry.IN_MEMORY_GUARDRAIL_HANDLER", mock_in_memory_handler, diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index 1d70126681d..9f7173383b0 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -60,3 +60,123 @@ def test_update_in_memory_guardrail(): handler.guardrail_id_to_custom_guardrail["123"].event_hook is GuardrailEventHooks.pre_call ) + + +def _make_guardrail(guardrail_id: str, name: str = "g") -> Guardrail: + return Guardrail( + guardrail_id=guardrail_id, + guardrail_name=name, + litellm_params=LitellmParams(guardrail=name, mode="pre_call", default_on=False), + ) + + +def test_reconcile_db_guardrails_drops_stale_db_entries_only(): + """ + The reconcile pass must drop in-memory entries marked source='db' that are + missing from the DB result, and never touch source='config' entries. + Models the multi-pod case where another pod deleted a DB-backed guardrail. + """ + handler = InMemoryGuardrailHandler() + + # Two DB-backed entries on this pod (synced from earlier polling cycles) + handler.IN_MEMORY_GUARDRAILS["db-keep"] = _make_guardrail("db-keep") + handler.IN_MEMORY_GUARDRAILS["db-stale"] = _make_guardrail("db-stale") + handler._sources["db-keep"] = "db" + handler._sources["db-stale"] = "db" + + # One config-loaded entry that must survive reconciliation + handler.IN_MEMORY_GUARDRAILS["cfg"] = _make_guardrail("cfg") + handler._sources["cfg"] = "config" + + # The DB now only contains db-keep — db-stale was deleted on another pod. + removed = handler.reconcile_db_guardrails(db_guardrail_ids={"db-keep"}) + + assert removed == ["db-stale"] + assert "db-stale" not in handler.IN_MEMORY_GUARDRAILS + assert "db-stale" not in handler._sources + assert "db-keep" in handler.IN_MEMORY_GUARDRAILS + assert "cfg" in handler.IN_MEMORY_GUARDRAILS + assert handler._sources["cfg"] == "config" + + +def test_reconcile_does_not_drop_config_entries_missing_from_db(): + """A config-only guardrail (no DB row) must never be reconciled away.""" + handler = InMemoryGuardrailHandler() + handler.IN_MEMORY_GUARDRAILS["cfg-only"] = _make_guardrail("cfg-only") + handler._sources["cfg-only"] = "config" + + removed = handler.reconcile_db_guardrails(db_guardrail_ids=set()) + + assert removed == [] + assert "cfg-only" in handler.IN_MEMORY_GUARDRAILS + + +def test_get_source_returns_marker_set_at_insert(): + handler = InMemoryGuardrailHandler() + handler.IN_MEMORY_GUARDRAILS["a"] = _make_guardrail("a") + handler._sources["a"] = "db" + handler.IN_MEMORY_GUARDRAILS["b"] = _make_guardrail("b") + handler._sources["b"] = "config" + + assert handler.get_source("a") == "db" + assert handler.get_source("b") == "config" + assert handler.get_source("missing") is None + + +def test_delete_in_memory_guardrail_clears_source_marker(): + handler = InMemoryGuardrailHandler() + handler.IN_MEMORY_GUARDRAILS["a"] = _make_guardrail("a") + handler._sources["a"] = "db" + + handler.delete_in_memory_guardrail("a") + + assert "a" not in handler.IN_MEMORY_GUARDRAILS + assert "a" not in handler._sources + assert handler.get_source("a") is None + + +def test_initialize_guardrail_early_return_updates_source_marker(): + """ + When initialize_guardrail is called for a guardrail that already exists + in memory, the early-return path must still honor the caller's source. + Otherwise a racing polling tick that placed a DB entry in memory first + would leave a later config-init call wrongly marked as 'db' (or vice + versa), and the entry would be reconciled with the wrong classification. + """ + handler = InMemoryGuardrailHandler() + # Simulate a polling tick already placing the entry as DB-backed. + handler.IN_MEMORY_GUARDRAILS["collide"] = _make_guardrail("collide", name="bedrock") + handler._sources["collide"] = "db" + + # Config init re-visits the same id (e.g., hot-reload, or UUID collision). + g = Guardrail( + guardrail_id="collide", + guardrail_name="bedrock", + litellm_params=LitellmParams( + guardrail="bedrock", mode="pre_call", default_on=False + ), + ) + handler.initialize_guardrail(guardrail=g, source="config") + + assert handler.get_source("collide") == "config" + + # And the symmetric direction: db sync should override an entry left + # marked as 'config' from a stale init path. + handler.initialize_guardrail(guardrail=g, source="db") + assert handler.get_source("collide") == "db" + + +def test_sync_guardrail_from_db_marks_source_db_when_unchanged(): + """ + sync_guardrail_from_db must enforce source='db' even when params are + unchanged, so a config entry whose UUID happens to collide with a later + DB row gets re-tagged correctly. + """ + handler = InMemoryGuardrailHandler() + g = _make_guardrail("collide") + handler.IN_MEMORY_GUARDRAILS["collide"] = g + handler._sources["collide"] = "config" + + handler.sync_guardrail_from_db(g) + + assert handler.get_source("collide") == "db" diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index 2edcb00c967..bcd7fcb37b3 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -466,6 +466,236 @@ async def test_test_model_connection_loads_config_from_router(): assert "result" in result +@pytest.mark.asyncio +async def test_test_model_connection_uses_model_info_id_to_disambiguate_duplicate_model_names(): + """ + When two deployments share the same `model_name` (e.g. wildcard + `openai/*`) but have different `api_base` values, clicking "Test + Connection" on a specific row in the UI must probe THAT row's + `api_base` — not whichever happens to be `deployments[0]`. + + The UI passes `model_info.id` to identify the deployment the user + actually clicked on. The backend must use that id to look up the + specific deployment rather than always grabbing the first match. + + Regression test for: silent fallback to deployments[0] when + multiple deployments share a wildcard model_name. + """ + from litellm.types.router import Deployment, LiteLLM_Params + + mock_request = MagicMock() + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "test-user" + mock_user_api_key_dict.token = "test-token" + + mock_prisma_client = MagicMock() + + deployment_a = { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_base": "https://deployment-A-base.invalid/v1", + "api_key": "fake-key-A", + }, + "model_info": {"id": "deployment-A-id"}, + } + deployment_b = { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_base": "https://deployment-B-base.invalid/v1", + "api_key": "fake-key-B", + }, + "model_info": {"id": "deployment-B-id"}, + } + + mock_router = MagicMock() + mock_router.get_model_list.return_value = [deployment_a, deployment_b] + + # Backend uses get_deployment(model_id=...) for O(1) lookup by id. + def _get_deployment_by_id(model_id): + if model_id == "deployment-A-id": + return Deployment( + model_name="openai/*", + litellm_params=LiteLLM_Params(**deployment_a["litellm_params"]), + model_info=deployment_a["model_info"], + ) + if model_id == "deployment-B-id": + return Deployment( + model_name="openai/*", + litellm_params=LiteLLM_Params(**deployment_b["litellm_params"]), + model_info=deployment_b["model_info"], + ) + return None + + mock_router.get_deployment.side_effect = _get_deployment_by_id + + mock_can_user_make_model_call = AsyncMock() + + mock_health_check_result = {"status": "healthy", "response_time_ms": 50} + mock_ahealth_check = AsyncMock(return_value=mock_health_check_result) + mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result) + + def mock_update_params(model_info, litellm_params): + params = litellm_params.copy() + params["messages"] = [{"role": "user", "content": "test"}] + return params + + def mock_reject_os_environ(params): + return None + + with ( + patch( + "litellm.proxy.proxy_server.prisma_client", + mock_prisma_client, + ), + patch( + "litellm.proxy.proxy_server.llm_router", + mock_router, + ), + patch( + "litellm.proxy.proxy_server.premium_user", + False, + ), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + mock_can_user_make_model_call, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints.litellm.ahealth_check", + mock_ahealth_check, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints.run_with_timeout", + mock_run_with_timeout, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints._update_litellm_params_for_health_check", + mock_update_params, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints._reject_os_environ_references", + mock_reject_os_environ, + ), + ): + # Click "Test Connection" on deployment B (NOT the first one). + # The UI sends only `model` + `model_info.id` — it does NOT + # send `api_base`/`api_key`, so the backend must resolve them + # from the right deployment. + await health_test_model_connection( + request=mock_request, + mode="chat", + litellm_params={"model": "openai/*"}, + model_info={"id": "deployment-B-id"}, + user_api_key_dict=mock_user_api_key_dict, + ) + + # The outbound health check must hit deployment B's api_base. + ahealth_check_call_args = mock_ahealth_check.call_args + assert ahealth_check_call_args is not None + model_params = ahealth_check_call_args.kwargs.get("model_params", {}) + + assert model_params.get("api_base") == ( + "https://deployment-B-base.invalid/v1" + ), ( + "Expected /health/test_connection to probe deployment B's " + "api_base when model_info.id='deployment-B-id' was provided. " + f"Got: {model_params.get('api_base')!r}. This means the " + "backend silently fell back to deployments[0] (A) instead " + "of disambiguating by model_info.id." + ) + assert model_params.get("api_key") == "fake-key-B" + + +@pytest.mark.asyncio +async def test_test_model_connection_falls_back_to_deployments_zero_without_id(): + """ + Backwards-compat: when the request body does NOT include + `model_info.id`, the legacy behavior of using `deployments[0]` + is preserved (single-deployment case, or callers that haven't + been updated to pass an id). + """ + mock_request = MagicMock() + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "test-user" + mock_user_api_key_dict.token = "test-token" + + mock_prisma_client = MagicMock() + + deployment_a = { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_base": "https://deployment-A-base.invalid/v1", + "api_key": "fake-key-A", + }, + "model_info": {"id": "deployment-A-id"}, + } + deployment_b = { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_base": "https://deployment-B-base.invalid/v1", + "api_key": "fake-key-B", + }, + "model_info": {"id": "deployment-B-id"}, + } + + mock_router = MagicMock() + mock_router.get_model_list.return_value = [deployment_a, deployment_b] + + mock_can_user_make_model_call = AsyncMock() + mock_health_check_result = {"status": "healthy"} + mock_ahealth_check = AsyncMock(return_value=mock_health_check_result) + mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result) + + def mock_update_params(model_info, litellm_params): + params = litellm_params.copy() + params["messages"] = [{"role": "user", "content": "test"}] + return params + + def mock_reject_os_environ(params): + return None + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client), + patch("litellm.proxy.proxy_server.llm_router", mock_router), + patch("litellm.proxy.proxy_server.premium_user", False), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + mock_can_user_make_model_call, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints.litellm.ahealth_check", + mock_ahealth_check, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints.run_with_timeout", + mock_run_with_timeout, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints._update_litellm_params_for_health_check", + mock_update_params, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints._reject_os_environ_references", + mock_reject_os_environ, + ), + ): + await health_test_model_connection( + request=mock_request, + mode="chat", + litellm_params={"model": "openai/*"}, + model_info={}, # no id provided + user_api_key_dict=mock_user_api_key_dict, + ) + + # Without id, deployments[0] (A) should be used (legacy behavior). + model_params = mock_ahealth_check.call_args.kwargs.get("model_params", {}) + assert model_params.get("api_base") == "https://deployment-A-base.invalid/v1" + assert model_params.get("api_key") == "fake-key-A" + + @pytest.mark.asyncio async def test_health_services_endpoint_datadog_llm_observability(): """ diff --git a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py index 39ec6f075d7..ee2d72d2dd4 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py @@ -380,6 +380,117 @@ async def test_list_tags_no_dynamic_tags(): app.dependency_overrides.clear() +@pytest.mark.asyncio +async def test_list_tags_with_date_range_filters_dynamic_tags(): + """ + /tag/list?start_date=...&end_date=... should push the date window into + the dailytagspend group_by WHERE clause so large tables don't get scanned. + """ + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + mock_user_auth = UserAPIKeyAuth( + user_id="test-user-123", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + + try: + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + mock_db = Mock() + mock_prisma.db = mock_db + mock_db.litellm_tagtable.find_many = AsyncMock(return_value=[]) + group_by_mock = AsyncMock(return_value=[]) + mock_db.litellm_dailytagspend.group_by = group_by_mock + + headers = {"Authorization": "Bearer sk-1234"} + response = client.get( + "/tag/list?start_date=2026-04-01&end_date=2026-04-29", + headers=headers, + ) + + assert response.status_code == 200 + group_by_mock.assert_awaited_once() + where = group_by_mock.await_args.kwargs["where"] + assert where["tag"] == {"not": None} + assert where["date"] == {"gte": "2026-04-01", "lte": "2026-04-29"} + + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_list_tags_without_date_range_omits_date_filter(): + """When no date range is passed, the WHERE clause must not carry a date key.""" + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + mock_user_auth = UserAPIKeyAuth( + user_id="test-user-123", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + + try: + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + mock_db = Mock() + mock_prisma.db = mock_db + mock_db.litellm_tagtable.find_many = AsyncMock(return_value=[]) + group_by_mock = AsyncMock(return_value=[]) + mock_db.litellm_dailytagspend.group_by = group_by_mock + + headers = {"Authorization": "Bearer sk-1234"} + response = client.get("/tag/list", headers=headers) + + assert response.status_code == 200 + group_by_mock.assert_awaited_once() + where = group_by_mock.await_args.kwargs["where"] + assert "date" not in where + + finally: + app.dependency_overrides.clear() + + +@pytest.mark.parametrize( + "query, expected_detail_fragment", + [ + ("?start_date=2026-04-01", "must be provided together"), + ("?end_date=2026-04-29", "must be provided together"), + ("?start_date=2026-04-29&end_date=2026-04-01", "on or before end_date"), + ("?start_date=not-a-date&end_date=2026-04-29", "YYYY-MM-DD"), + ], +) +@pytest.mark.asyncio +async def test_list_tags_rejects_invalid_date_range(query, expected_detail_fragment): + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + mock_user_auth = UserAPIKeyAuth( + user_id="test-user-123", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + + try: + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + mock_db = Mock() + mock_prisma.db = mock_db + mock_db.litellm_tagtable.find_many = AsyncMock(return_value=[]) + mock_db.litellm_dailytagspend.group_by = AsyncMock(return_value=[]) + + headers = {"Authorization": "Bearer sk-1234"} + response = client.get(f"/tag/list{query}", headers=headers) + + assert response.status_code == 400 + assert expected_detail_fragment in response.json()["detail"] + + finally: + app.dependency_overrides.clear() + + @pytest.mark.asyncio async def test_get_deployments_by_model_id(): """ diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx index 809f1d4e17b..efc1166a398 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx @@ -145,23 +145,6 @@ const UsagePage: React.FC = ({ teams, organizations }) => { const [topKeysLimit, setTopKeysLimit] = useState(5); const [topModelsLimit, setTopModelsLimit] = useState(5); const [showTokenBreakdown, setShowTokenBreakdown] = useState(false); - const getAllTags = async () => { - if (!accessToken) { - return; - } - const tags = await tagListCall(accessToken); - setAllTags( - Object.values(tags).map((tag: Tag) => ({ - label: tag.name, - value: tag.name, - })), - ); - }; - - useEffect(() => { - getAllTags(); - }, [accessToken]); - // Sync selectedUserId when auth state settles (isAdmin/userID may be null on initial render) useEffect(() => { if (!isAdmin && userID) { @@ -175,6 +158,30 @@ const UsagePage: React.FC = ({ teams, organizations }) => { const startTime = useMemo(() => (dateValue.from ? new Date(dateValue.from) : null), [dateValue.from]); const endTime = useMemo(() => (dateValue.to ? new Date(dateValue.to) : null), [dateValue.to]); + useEffect(() => { + if (!accessToken) return; + let cancelled = false; + (async () => { + try { + const tags = await tagListCall(accessToken, startTime, endTime); + if (cancelled) return; + setAllTags( + Object.values(tags).map((tag: Tag) => ({ + label: tag.name, + value: tag.name, + })), + ); + } catch (e) { + if (!cancelled) { + console.error("Failed to fetch tag list", e); + } + } + })(); + return () => { + cancelled = true; + }; + }, [accessToken, startTime, endTime]); + // Try aggregated endpoint first, fall back to paginated on failure const aggregatedFetchIdRef = useRef(0); useEffect(() => { diff --git a/ui/litellm-dashboard/src/components/model_info_view.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.test.tsx index 29eb8e0019b..54144d02eda 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.test.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.test.tsx @@ -250,6 +250,33 @@ describe("ModelInfoView", () => { }); }); + it("should pass model_info.id to disambiguate duplicate model_name deployments", async () => { + // Regression test: when two deployments share `model_name` (e.g. + // wildcard `openai/*` with different `api_base` values), the UI + // must forward the clicked row's `model_info.id` to the backend. + // Otherwise /health/test_connection silently probes deployments[0] + // instead of the deployment the user actually selected. + const user = userEvent.setup(); + render(, { wrapper }); + + await waitFor(() => { + expect(screen.getByText("Model Settings")).toBeInTheDocument(); + }); + + const testButton = screen.getByRole("button", { name: /test connection/i }); + await user.click(testButton); + + await waitFor(() => { + expect(mockTestConnectionRequest).toHaveBeenCalled(); + }); + + const callArgs = mockTestConnectionRequest.mock.calls[0]; + // Signature: (accessToken, litellm_params, model_info, mode) + const modelInfoArg = callArgs[2] as Record; + expect(modelInfoArg).toBeDefined(); + expect(modelInfoArg.id).toBe("123"); + }); + it("should display error notification when connection test fails", async () => { const user = userEvent.setup(); mockTestConnectionRequest.mockRejectedValue(new Error("Connection failed")); diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index ea8af2fcd62..95a43862de5 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -379,6 +379,12 @@ export default function ModelInfoView({ model: localModelData.litellm_model_name, }, { + // `id` is required to disambiguate when multiple deployments + // share the same model_name (e.g. wildcard `openai/*` with two + // different `api_base` values for failover). Without it the + // backend silently falls back to deployments[0] and probes + // the wrong endpoint. + id: localModelData.model_info?.id, mode: localModelData.model_info?.mode, }, localModelData.model_info?.mode, diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 44208904a70..abe17616bef 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -7288,10 +7288,29 @@ export const tagInfoCall = async (accessToken: string, tagNames: string[]): Prom } }; -export const tagListCall = async (accessToken: string): Promise => { +const formatYmd = (value: Date): string => { + const year = value.getFullYear(); + const month = String(value.getMonth() + 1).padStart(2, "0"); + const day = String(value.getDate()).padStart(2, "0"); + return `${year}-${month}-${day}`; +}; + +export const tagListCall = async ( + accessToken: string, + startTime?: Date | null, + endTime?: Date | null, +): Promise => { try { let url = proxyBaseUrl ? `${proxyBaseUrl}/tag/list` : `/tag/list`; + if (startTime && endTime) { + const params = new URLSearchParams({ + start_date: formatYmd(startTime), + end_date: formatYmd(endTime), + }); + url = `${url}?${params.toString()}`; + } + const response = await fetch(url, { method: "GET", headers: { diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 492e43cbc81..65bd9d9eb95 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -396,7 +396,7 @@ export default function KeyInfoView({ }; return ( -

+
- +
Key Settings {!isEditing && canModifyKey && ( From bac03ac3f162c752457990430bf8461606aae03e Mon Sep 17 00:00:00 2001 From: milan-berri Date: Fri, 8 May 2026 00:34:49 +0300 Subject: [PATCH 26/85] feat(auth): add scope and wildcard support for JWT routing overrides (#26325) Squash-merged by litellm-agent from milan-berri's PR. --- litellm/proxy/_types.py | 6 + litellm/proxy/auth/user_api_key_auth.py | 41 ++- .../proxy/auth/test_user_api_key_auth.py | 335 +++++++++++++++++- 3 files changed, 377 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ed20fe86cdc..89b422bd5d0 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4342,10 +4342,16 @@ class JWTRoutingOverride(BaseModel): A rule matches when all provided selectors match token claims. If matched, request is routed to the configured auth path. + + Wildcard selectors use shell-style patterns (* and ?) and are matched with + case-sensitive semantics; use the same casing your IdP emits in JWT claims. + Space-delimited tokenization applies only to the ``scope`` claim (OAuth/OIDC + scope strings), not to ``iss``, ``aud``, or ``client_id``. """ iss: Union[str, List[str]] client_id: Optional[Union[str, List[str]]] = None + scope: Optional[Union[str, List[str]]] = None aud: Optional[Union[str, List[str]]] = None path: Literal["oauth2"] = "oauth2" diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 9d3c06e641f..2f61a40bc9a 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -8,6 +8,7 @@ Returns a UserAPIKeyAuth object if the API key is valid """ import asyncio +import fnmatch import re import secrets from datetime import datetime, timezone @@ -183,22 +184,49 @@ def _get_bearer_token_or_received_api_key(api_key: str) -> str: def _routing_selector_matches_claim( - selector_value: Optional[Any], claim_value: Optional[Any] + selector_value: Optional[Any], + claim_value: Optional[Any], + *, + split_space_delimited: bool = False, ) -> bool: if selector_value is None: return True - selector_list = ( + selector_list: List[str] = ( [str(v) for v in selector_value] if isinstance(selector_value, list) else [str(selector_value)] ) + if claim_value is None: + return False + if isinstance(claim_value, list): claim_list = [str(v) for v in claim_value] - return any(v in claim_list for v in selector_list) + elif ( + split_space_delimited + and isinstance(claim_value, str) + and " " in claim_value.strip() + ): + # OAuth/OIDC often sends scope as a single space-delimited string. Only split + # for the scope selector: iss/aud/client_id must stay exact full-string match + # on unverified claims (see routing override security review). The elif guard + # (`" " in claim_value.strip()`) ensures at least two non-empty tokens survive. + claim_list = [v for v in claim_value.strip().split(" ") if v] + else: + claim_list = [str(claim_value)] - return str(claim_value) in selector_list if claim_value is not None else False + def _selector_matches_claim(selector: str, claim: str) -> bool: + # NOTE: wildcard matching is case-sensitive (fnmatch.fnmatchcase). + if "*" in selector or "?" in selector: + return fnmatch.fnmatchcase(claim, selector) + return selector == claim + + return any( + _selector_matches_claim(selector=s, claim=c) + for s in selector_list + for c in claim_list + ) def _matches_routing_override( @@ -209,6 +237,11 @@ def _matches_routing_override( and _routing_selector_matches_claim( override.client_id, token_claims.get("client_id") ) + and _routing_selector_matches_claim( + override.scope, + token_claims.get("scope"), + split_space_delimited=True, + ) and _routing_selector_matches_claim(override.aud, token_claims.get("aud")) ) diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 95b3d746c66..4b0f6c0ea2f 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -29,8 +29,10 @@ from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.auth_checks import get_key_object, _cache_key_object from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import ( - _route_requires_auth_despite_public, + _matches_routing_override, _reserve_budget_after_common_checks, + _route_requires_auth_despite_public, + _routing_selector_matches_claim, _run_centralized_common_checks, _run_post_custom_auth_checks, get_api_key, @@ -572,6 +574,140 @@ def _assert_get_api_key_with_custom_litellm_key_header( ) == (api_key, passed_in_key) +@pytest.mark.parametrize( + "selector_value, claim_value, expected, split_space_delimited", + [ + (None, "any-value", True, False), + ("issuer.example.com", "issuer.example.com", True, False), + ("issuer.example.com", "other-issuer.example.com", False, False), + # iss (and other non-scope claims) must not match via space-split injection + ( + "trusted.example.com", + "trusted.example.com attacker.example.com", + False, + False, + ), + ( + ["issuer-a.example.com", "issuer-b.example.com"], + "issuer-b.example.com", + True, + False, + ), + ("*MID_LITELLM", "STREAM_MID_LITELLM", True, False), + ("*MID_LITELLM", "REDIS_LITELLM", False, False), + ("machine-??", "machine-01", True, False), + ("machine-??", "machine-001", False, False), + # Wildcard matching is case-sensitive (fnmatch.fnmatchcase) + ("*litellm", "BATCH_LITELLM", False, False), + ("*LITELLM", "BATCH_LITELLM", True, False), + ("App:LiteLLM", "App:LiteLLM openid", True, True), + ("App:*", "App:LiteLLM openid", True, True), + (["openid", "App:LiteLLM"], "openid profile", True, True), + (["service-*", "batch-*"], "batch-123", True, False), + (["service-*", "batch-*"], "other-123", False, False), + ("App:LiteLLM", ["openid", "App:LiteLLM"], True, False), + ("App:LiteLLM", None, False, False), + ], +) +def test_routing_selector_matches_claim_parametrized( + selector_value, claim_value, expected, split_space_delimited +): + assert ( + _routing_selector_matches_claim( + selector_value=selector_value, + claim_value=claim_value, + split_space_delimited=split_space_delimited, + ) + is expected + ) + + +@pytest.mark.parametrize( + "override, token_claims, expected", + [ + # Only iss selector is required and should match. + ( + JWTRoutingOverride(iss="oauth-issuer.example.com", path="oauth2"), + {"iss": "oauth-issuer.example.com"}, + True, + ), + # Scope selector narrows the match. + ( + JWTRoutingOverride( + iss="oauth-issuer.example.com", + scope="App:LiteLLM", + path="oauth2", + ), + {"iss": "oauth-issuer.example.com", "scope": "App:LiteLLM openid"}, + True, + ), + # client_id wildcard selector narrows the match. + ( + JWTRoutingOverride( + iss="oauth-issuer.example.com", + client_id="*MID_LITELLM", + path="oauth2", + ), + {"iss": "oauth-issuer.example.com", "client_id": "BATCH_MID_LITELLM"}, + True, + ), + ( + JWTRoutingOverride( + iss="oauth-issuer.example.com", + client_id="*MID_LITELLM", + path="oauth2", + ), + {"iss": "oauth-issuer.example.com", "client_id": "BATCH_PORTAL"}, + False, + ), + # aud selector still works with list claims. + ( + JWTRoutingOverride( + iss="oauth-issuer.example.com", + aud=["api://litellm", "api://fallback"], + path="oauth2", + ), + {"iss": "oauth-issuer.example.com", "aud": ["api://other", "api://litellm"]}, + True, + ), + # All provided selectors are AND-ed. + ( + JWTRoutingOverride( + iss="oauth-issuer.example.com", + scope="App:LiteLLM", + client_id="*MID_LITELLM", + path="oauth2", + ), + { + "iss": "oauth-issuer.example.com", + "scope": "App:LiteLLM openid", + "client_id": "BATCH_MID_LITELLM", + }, + True, + ), + ( + JWTRoutingOverride( + iss="oauth-issuer.example.com", + scope="App:LiteLLM", + client_id="*MID_LITELLM", + path="oauth2", + ), + { + "iss": "oauth-issuer.example.com", + "scope": "App:Other openid", + "client_id": "BATCH_MID_LITELLM", + }, + False, + ), + ], +) +def test_matches_routing_override_parametrized(override, token_claims, expected): + assert ( + _matches_routing_override(token_claims=token_claims, override=override) + is expected + ) + + def test_get_api_key_with_custom_litellm_key_header_bearer_prefix(): token = "sk-" + "1" * 8 header = f"Bearer {token}" @@ -1578,6 +1714,203 @@ class TestJWTOAuth2Coexistence: mock_jwt_auth.assert_not_called() assert result.user_id == "machine-client-aud-list" + @pytest.mark.asyncio + async def test_routing_override_matches_scope_claim(self): + """ + Match routing override when scope selector is configured and scope claim matches. + """ + jwt_token = ( + "eyJhbGciOiJSUzI1NiJ9." + "eyJpc3MiOiJvYXV0aC1pc3N1ZXIuZXhhbXBsZS5jb20iLCJzY29wZSI6IkFwcDpMaXRlTExNIiwiY2xpZW50X2lkIjoiTUFDSElORV9NSURfTElURUxMTSJ9." + "c2ln" + ) + general_settings = { + "enable_oauth2_auth": False, + "enable_jwt_auth": True, + } + mock_oauth2_response = UserAPIKeyAuth( + api_key=jwt_token, + user_id="machine-client-scope-match", + ) + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + return_value=mock_oauth2_response, + ) as mock_oauth2, patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + ) as mock_jwt_auth: + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth( + routing_overrides=[ + JWTRoutingOverride( + iss="oauth-issuer.example.com", + scope="App:LiteLLM", + path="oauth2", + ) + ] + ), + ) + + result = await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {jwt_token}", + ) + + mock_oauth2.assert_called_once_with(token=jwt_token) + mock_jwt_auth.assert_not_called() + assert result.user_id == "machine-client-scope-match" + + @pytest.mark.asyncio + async def test_routing_override_scope_mismatch_falls_back_to_jwt(self): + """ + If scope selector does not match, continue default JWT flow. + """ + jwt_token = ( + "eyJhbGciOiJSUzI1NiJ9." + "eyJpc3MiOiJvYXV0aC1pc3N1ZXIuZXhhbXBsZS5jb20iLCJzY29wZSI6IkFwcDpPdGhlciIsImNsaWVudF9pZCI6IlBPUlRBTF9NSURfTElURUxMTSJ9." + "c2ln" + ) + general_settings = { + "enable_oauth2_auth": False, + "enable_jwt_auth": True, + } + mock_jwt_result = { + "is_proxy_admin": True, + "team_object": None, + "user_object": None, + "end_user_object": None, + "org_object": None, + "token": jwt_token, + "team_id": "jwt-team", + "user_id": "jwt-user-scope-mismatch", + "end_user_id": None, + "org_id": None, + "team_membership": None, + "jwt_claims": {"sub": "user1"}, + } + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + ) as mock_oauth2, patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + return_value=mock_jwt_result, + ) as mock_jwt_auth: + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth( + routing_overrides=[ + JWTRoutingOverride( + iss="oauth-issuer.example.com", + scope="App:LiteLLM", + path="oauth2", + ) + ] + ), + ) + + result = await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {jwt_token}", + ) + + mock_oauth2.assert_not_called() + mock_jwt_auth.assert_called_once() + assert result.user_id == "jwt-user-scope-mismatch" + + @pytest.mark.asyncio + async def test_routing_override_matches_scope_and_client_wildcard_when_scope_claim_is_space_delimited( + self, + ): + """ + Integration check: combined scope + wildcard selectors match on OAuth2 path + when scope claim is a space-delimited string. + """ + jwt_token = ( + "eyJhbGciOiJSUzI1NiJ9." + "eyJpc3MiOiJvYXV0aC1pc3N1ZXIuZXhhbXBsZS5jb20iLCJzY29wZSI6IkFwcDpMaXRlTExNIG9wZW5pZCIsImNsaWVudF9pZCI6IkJBVENIX01JRF9MSVRFTExNIn0." + "c2ln" + ) + general_settings = { + "enable_oauth2_auth": False, + "enable_jwt_auth": True, + } + mock_oauth2_response = UserAPIKeyAuth( + api_key=jwt_token, + user_id="machine-client-space-delimited-scope-match", + ) + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + return_value=mock_oauth2_response, + ) as mock_oauth2, patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + ) as mock_jwt_auth: + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth( + routing_overrides=[ + JWTRoutingOverride( + iss="oauth-issuer.example.com", + scope="App:LiteLLM", + client_id="*MID_LITELLM", + path="oauth2", + ) + ] + ), + ) + + result = await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {jwt_token}", + ) + + mock_oauth2.assert_called_once_with(token=jwt_token) + mock_jwt_auth.assert_not_called() + assert result.user_id == "machine-client-space-delimited-scope-match" + @pytest.mark.asyncio async def test_routing_override_routes_jwt_to_oauth2_when_oauth2_globally_disabled( self, From f58f8927f27ced15d9e27dcb46cfef530d670f52 Mon Sep 17 00:00:00 2001 From: shivam Date: Thu, 7 May 2026 18:39:26 -0700 Subject: [PATCH 27/85] feat(guardrails): optional skip tool message in unified guardrail inputs Mirrors the system-message skip in PR #25481 for tool-role messages. Adds a global litellm.skip_tool_message_in_guardrail flag and a per-guardrail litellm_params.skip_tool_message_in_guardrail override, applied in the OpenAI and Anthropic chat translation handlers. Co-Authored-By: Claude Opus 4.7 (1M context) --- litellm/__init__.py | 1 + .../chat/guardrail_translation/handler.py | 12 +- .../base_llm/guardrail_translation/utils.py | 15 ++ .../chat/guardrail_translation/handler.py | 24 +++- .../proxy/guardrails/guardrail_registry.py | 5 + litellm/types/guardrails.py | 10 ++ .../test_unified_guardrail.py | 132 ++++++++++++++++++ 7 files changed, 192 insertions(+), 7 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index cf05fc4c980..fd3d47ec154 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -206,6 +206,7 @@ add_user_information_to_llm_headers: Optional[bool] = ( ) store_audit_logs = False # Enterprise feature, allow users to see audit logs skip_system_message_in_guardrail: bool = False +skip_tool_message_in_guardrail: bool = False ### end of callbacks ############# email: Optional[str] = ( diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 2bb82f227bb..74dadee5ecb 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -23,7 +23,9 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, openai_messages_without_system, + openai_messages_without_tool, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, @@ -108,6 +110,7 @@ class AnthropicMessagesHandler(BaseTranslation): return data skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply) + skip_tool = effective_skip_tool_message_for_guardrail(guardrail_to_apply) chat_completion_compatible_request = self._translate_to_openai(data) @@ -117,6 +120,8 @@ class AnthropicMessagesHandler(BaseTranslation): ) if skip_system: structured_messages = openai_messages_without_system(structured_messages) + if skip_tool: + structured_messages = openai_messages_without_tool(structured_messages) texts_to_check: List[str] = [] images_to_check: List[str] = [] @@ -134,6 +139,7 @@ class AnthropicMessagesHandler(BaseTranslation): images_to_check=images_to_check, task_mappings=task_mappings, skip_system_message=skip_system, + skip_tool_message=skip_tool, ) # Step 2: Apply guardrail to all texts in batch @@ -198,13 +204,17 @@ class AnthropicMessagesHandler(BaseTranslation): images_to_check: List[str], task_mappings: List[Tuple[int, Optional[int]]], skip_system_message: bool = False, + skip_tool_message: bool = False, ) -> None: """ Extract text content and images from a message. Override this method to customize text/image extraction logic. """ - if skip_system_message and str(message.get("role") or "").lower() == "system": + role = str(message.get("role") or "").lower() + if skip_system_message and role == "system": + return + if skip_tool_message and role == "tool": return content = message.get("content", None) diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index cdd2d775371..97ece6b5eab 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -14,7 +14,22 @@ def effective_skip_system_message_for_guardrail(guardrail_to_apply: Any) -> bool return bool(getattr(litellm, "skip_system_message_in_guardrail", False)) +def effective_skip_tool_message_for_guardrail(guardrail_to_apply: Any) -> bool: + per = getattr(guardrail_to_apply, "skip_tool_message_in_guardrail", None) + if per is not None: + return bool(per) + import litellm + + return bool(getattr(litellm, "skip_tool_message_in_guardrail", False)) + + def openai_messages_without_system( messages: List[AllMessageValues], ) -> List[AllMessageValues]: return [m for m in messages if str((m or {}).get("role") or "").lower() != "system"] + + +def openai_messages_without_tool( + messages: List[AllMessageValues], +) -> List[AllMessageValues]: + return [m for m in messages if str((m or {}).get("role") or "").lower() != "tool"] diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 86ca6625629..d413a244539 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -21,7 +21,9 @@ from litellm._logging import verbose_proxy_logger from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, openai_messages_without_system, + openai_messages_without_tool, ) from litellm.main import stream_chunk_builder from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam @@ -73,6 +75,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return data skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply) + skip_tool = effective_skip_tool_message_for_guardrail(guardrail_to_apply) texts_to_check: List[str] = [] images_to_check: List[str] = [] @@ -91,6 +94,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): text_task_mappings=text_task_mappings, tool_call_task_mappings=tool_call_task_mappings, skip_system_message=skip_system, + skip_tool_message=skip_tool, ) # Step 2: Apply guardrail to all texts and tool calls in batch @@ -102,11 +106,15 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs["tool_calls"] = tool_calls_to_check # type: ignore structured_messages = self.get_structured_messages(data) if structured_messages: - inputs["structured_messages"] = ( - openai_messages_without_system(structured_messages) - if skip_system - else structured_messages - ) + if skip_system: + structured_messages = openai_messages_without_system( + structured_messages + ) + if skip_tool: + structured_messages = openai_messages_without_tool( + structured_messages + ) + inputs["structured_messages"] = structured_messages # Pass tools (function definitions) to the guardrail tools = data.get("tools") if tools: @@ -176,13 +184,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation): text_task_mappings: List[Tuple[int, Optional[int]]], tool_call_task_mappings: List[Tuple[int, int]], skip_system_message: bool = False, + skip_tool_message: bool = False, ) -> None: """ Extract text content, images, and tool calls from a message. Override this method to customize text/image/tool call extraction logic. """ - if skip_system_message and str(message.get("role") or "").lower() == "system": + role = str(message.get("role") or "").lower() + if skip_system_message and role == "system": + return + if skip_tool_message and role == "tool": return content = message.get("content", None) diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 868b23756d2..838fb2e01ad 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -482,6 +482,11 @@ class InMemoryGuardrailHandler: "skip_system_message_in_guardrail", getattr(litellm_params, "skip_system_message_in_guardrail", None), ) + setattr( + custom_guardrail_callback, + "skip_tool_message_in_guardrail", + getattr(litellm_params, "skip_tool_message_in_guardrail", None), + ) parsed_guardrail = Guardrail( guardrail_id=guardrail.get("guardrail_id"), diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 04347aebe3b..751113400d3 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -633,6 +633,16 @@ class BaseLitellmParams( ), ) + skip_tool_message_in_guardrail: Optional[bool] = Field( + default=None, + description=( + "When True, unified guardrails skip tool-role messages when building " + "evaluation inputs (texts and structured_messages). When False, tool " + "messages are included even if litellm_settings sets a global skip. When " + "None, use the global litellm.skip_tool_message_in_guardrail setting." + ), + ) + # Lakera specific params category_thresholds: Optional[LakeraCategoryThresholds] = Field( default=None, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 2418d7af04b..6e027fa4941 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -8,7 +8,9 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, openai_messages_without_system, + openai_messages_without_tool, ) from litellm.llms.openai.chat.guardrail_translation.handler import ( OpenAIChatCompletionsHandler, @@ -180,6 +182,136 @@ class TestUnifiedLLMGuardrails: } assert "system" in roles + class TestSkipToolMessageForChatCompletions: + def test_openai_messages_without_tool(self): + msgs = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "content": "tool result", "tool_call_id": "call_1"}, + ] + out = openai_messages_without_tool(msgs) + assert len(out) == 2 + assert all(m["role"] != "tool" for m in out) + assert msgs[2]["content"] == "tool result" + + def test_effective_skip_tool_respects_per_guardrail_over_global( + self, monkeypatch + ): + monkeypatch.setattr( + litellm, "skip_tool_message_in_guardrail", True, raising=False + ) + + class G: + skip_tool_message_in_guardrail = False + + assert effective_skip_tool_message_for_guardrail(G()) is False + + class G2: + skip_tool_message_in_guardrail = None + + assert effective_skip_tool_message_for_guardrail(G2()) is True + + @pytest.mark.asyncio + async def test_openai_handler_skips_tool_in_guardrail_inputs(self, monkeypatch): + monkeypatch.setattr( + litellm, "skip_tool_message_in_guardrail", True, raising=False + ) + + captured = {} + + class MockGuardrail: + skip_tool_message_in_guardrail = None + + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ): + captured["inputs"] = inputs + return inputs + + data = { + "messages": [ + {"role": "user", "content": "hello"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + } + ], + }, + { + "role": "tool", + "content": "secret tool result", + "tool_call_id": "call_1", + }, + ], + "model": "gpt-4o", + } + + handler = OpenAIChatCompletionsHandler() + await handler.process_input_messages( + data=data, + guardrail_to_apply=MockGuardrail(), + litellm_logging_obj=None, + ) + + assert "secret tool result" not in captured["inputs"]["texts"] + sm = captured["inputs"].get("structured_messages") or [] + assert all(m.get("role") != "tool" for m in sm) + assert data["messages"][2]["content"] == "secret tool result" + + @pytest.mark.asyncio + async def test_openai_handler_per_guardrail_skip_tool_false_overrides_global( + self, monkeypatch + ): + monkeypatch.setattr( + litellm, "skip_tool_message_in_guardrail", True, raising=False + ) + + captured = {} + + class MockGuardrail: + skip_tool_message_in_guardrail = False + + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ): + captured["inputs"] = inputs + return inputs + + data = { + "messages": [ + {"role": "user", "content": "u"}, + {"role": "tool", "content": "tr", "tool_call_id": "call_1"}, + ], + } + + await OpenAIChatCompletionsHandler().process_input_messages( + data=data, + guardrail_to_apply=MockGuardrail(), + litellm_logging_obj=None, + ) + + assert "tr" in captured["inputs"]["texts"] + roles = { + m.get("role") + for m in (captured["inputs"].get("structured_messages") or []) + } + assert "tool" in roles + class TestAsyncPreCallHook: @pytest.mark.asyncio async def test_uses_mcp_event_type(self): From bdb2b0e708a032d6235b934917d297168f7bd9e1 Mon Sep 17 00:00:00 2001 From: shivam Date: Thu, 7 May 2026 19:11:41 -0700 Subject: [PATCH 28/85] feat(dashboard): skip_tool_message_in_guardrail in guardrail UI Adds a tri-state control (inherit / yes / no) when creating or editing guardrails so admins can set litellm_params.skip_tool_message_in_guardrail without YAML, mirroring the existing skip_system_message control. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../guardrails/add_guardrail_form.tsx | 20 +++++++++++ .../guardrails/edit_guardrail_form.tsx | 23 ++++++++++++ .../components/guardrails/guardrail_info.tsx | 36 +++++++++++++++++++ .../guardrail_info_helpers.test.tsx | 16 +++++++++ .../guardrails/guardrail_info_helpers.tsx | 16 +++++++++ .../components/guardrails/guardrail_table.tsx | 10 +++++- 6 files changed, 120 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx index c91a6e85cdd..16c1c6efecd 100644 --- a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx @@ -5,6 +5,7 @@ import { createGuardrailCall, getGuardrailProviderSpecificParams, getGuardrailUI import ContentFilterConfiguration from "./content_filter/ContentFilterConfiguration"; import { choiceToSkipSystemForCreate, + choiceToSkipToolForCreate, getGuardrailProviders, guardrail_provider_map, guardrailLogoMap, @@ -188,6 +189,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a mode: preset.mode, default_on: preset.defaultOn, skip_system_message_choice: "inherit", + skip_tool_message_choice: "inherit", }; if (preset.provider === "BlockCodeExecution") { baseValues.confidence_threshold = 0.5; @@ -433,6 +435,11 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a guardrailData.litellm_params.skip_system_message_in_guardrail = skipForCreate; } + const skipToolForCreate = choiceToSkipToolForCreate(values.skip_tool_message_choice); + if (skipToolForCreate !== undefined) { + guardrailData.litellm_params.skip_tool_message_in_guardrail = skipToolForCreate; + } + // For Presidio PII, add the entity and action configurations if (values.provider === "PresidioPII" && selectedEntities.length > 0) { const piiEntitiesConfig: { [key: string]: string } = {}; @@ -804,6 +811,18 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a + + + + {/* Use the GuardrailProviderFields component to render provider-specific fields */} {!isToolPermissionProvider && !shouldRenderContentFilterConfigSettings(selectedProvider) && !shouldRenderLLMJudgeFields(selectedProvider) && ( = ({ visible, onClose, a mode: "pre_call", default_on: false, skip_system_message_choice: "inherit", + skip_tool_message_choice: "inherit", }} > {stepConfigs.map((step, index) => { diff --git a/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx b/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx index ad823df53fc..8ba9b0b312f 100644 --- a/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx @@ -6,6 +6,7 @@ import { guardrailLogoMap, getGuardrailProviders, type SkipSystemMessageChoice, + type SkipToolMessageChoice, } from "./guardrail_info_helpers"; import { getGuardrailUISettings, getGlobalLitellmHeaderName } from "../networking"; import PiiConfiguration from "./pii_configuration"; @@ -29,6 +30,7 @@ interface EditGuardrailFormProps { default_on: boolean; pii_entities_config?: { [key: string]: string }; skip_system_message_choice?: SkipSystemMessageChoice; + skip_tool_message_choice?: SkipToolMessageChoice; [key: string]: any; }; } @@ -138,6 +140,15 @@ const EditGuardrailForm: React.FC = ({ delete litellm_params.skip_system_message_in_guardrail; } + const skipToolChoice = values.skip_tool_message_choice as SkipToolMessageChoice | undefined; + if (skipToolChoice === "yes") { + litellm_params.skip_tool_message_in_guardrail = true; + } else if (skipToolChoice === "no") { + litellm_params.skip_tool_message_in_guardrail = false; + } else { + delete litellm_params.skip_tool_message_in_guardrail; + } + let guardrail_info: any = {}; // For Presidio PII, add the entity and action configurations @@ -432,6 +443,18 @@ const EditGuardrailForm: React.FC = ({ + + + + {renderProviderSpecificFields()}
diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx index 60400443d5c..53aebcff0de 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx @@ -29,7 +29,9 @@ import { getGuardrailLogoAndName, guardrail_provider_map, skipSystemMessageToChoice, + skipToolMessageToChoice, type SkipSystemMessageChoice, + type SkipToolMessageChoice, } from "./guardrail_info_helpers"; import GuardrailOptionalParams from "./guardrail_optional_params"; import GuardrailProviderFields from "./guardrail_provider_fields"; @@ -214,12 +216,16 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, if (guardrailData && form) { const lp = { ...(guardrailData.litellm_params || {}) }; delete lp.skip_system_message_in_guardrail; + delete lp.skip_tool_message_in_guardrail; form.setFieldsValue({ guardrail_name: guardrailData.guardrail_name, ...lp, skip_system_message_choice: skipSystemMessageToChoice( guardrailData.litellm_params?.skip_system_message_in_guardrail, ), + skip_tool_message_choice: skipToolMessageToChoice( + guardrailData.litellm_params?.skip_tool_message_in_guardrail, + ), guardrail_info: guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : "", // Include any optional_params if they exist ...(guardrailData.litellm_params?.optional_params && { @@ -302,6 +308,20 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, } } + const prevSkipToolChoice = skipToolMessageToChoice( + guardrailData.litellm_params?.skip_tool_message_in_guardrail, + ); + const nextSkipToolChoice = values.skip_tool_message_choice as SkipToolMessageChoice | undefined; + if (nextSkipToolChoice !== undefined && nextSkipToolChoice !== prevSkipToolChoice) { + if (nextSkipToolChoice === "inherit") { + updateData.litellm_params.skip_tool_message_in_guardrail = null; + } else if (nextSkipToolChoice === "yes") { + updateData.litellm_params.skip_tool_message_in_guardrail = true; + } else { + updateData.litellm_params.skip_tool_message_in_guardrail = false; + } + } + // Only include guardrail_info if it has changed const originalGuardrailInfo = guardrailData.guardrail_info; const newGuardrailInfo = values.guardrail_info ? JSON.parse(values.guardrail_info) : undefined; @@ -674,11 +694,15 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, ...(() => { const lp = { ...(guardrailData.litellm_params || {}) }; delete lp.skip_system_message_in_guardrail; + delete lp.skip_tool_message_in_guardrail; return lp; })(), skip_system_message_choice: skipSystemMessageToChoice( guardrailData.litellm_params?.skip_system_message_in_guardrail, ), + skip_tool_message_choice: skipToolMessageToChoice( + guardrailData.litellm_params?.skip_tool_message_in_guardrail, + ), guardrail_info: guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : "", @@ -716,6 +740,18 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, + + + + {guardrailData.litellm_params?.guardrail === "presidio" && ( <> PII Protection diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx index dfda86c1e4a..1fc62f94cf1 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx @@ -12,6 +12,8 @@ import { GuardrailProviders, skipSystemMessageToChoice, choiceToSkipSystemForCreate, + skipToolMessageToChoice, + choiceToSkipToolForCreate, } from "./guardrail_info_helpers"; describe("guardrail_info_helpers", () => { @@ -215,4 +217,18 @@ describe("guardrail_info_helpers", () => { expect(choiceToSkipSystemForCreate("no")).toBe(false); }); }); + + describe("skipToolMessageToChoice / choiceToSkipToolForCreate", () => { + it("maps API values to form choices and back for create", () => { + expect(skipToolMessageToChoice(undefined)).toBe("inherit"); + expect(skipToolMessageToChoice(null)).toBe("inherit"); + expect(skipToolMessageToChoice(true)).toBe("yes"); + expect(skipToolMessageToChoice(false)).toBe("no"); + + expect(choiceToSkipToolForCreate("inherit")).toBeUndefined(); + expect(choiceToSkipToolForCreate(undefined)).toBeUndefined(); + expect(choiceToSkipToolForCreate("yes")).toBe(true); + expect(choiceToSkipToolForCreate("no")).toBe(false); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx index ac4b787e96a..54b16b81765 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx @@ -179,3 +179,19 @@ export function choiceToSkipSystemForCreate(choice: SkipSystemMessageChoice | un if (choice === "no") return false; return undefined; } + +/** Tri-state UI value for `litellm_params.skip_tool_message_in_guardrail` (inherit = use global). */ +export type SkipToolMessageChoice = "inherit" | "yes" | "no"; + +export function skipToolMessageToChoice(v: boolean | null | undefined): SkipToolMessageChoice { + if (v === true) return "yes"; + if (v === false) return "no"; + return "inherit"; +} + +/** Create flow: omit key when inheriting global default. */ +export function choiceToSkipToolForCreate(choice: SkipToolMessageChoice | undefined): boolean | undefined { + if (choice === "yes") return true; + if (choice === "no") return false; + return undefined; +} diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx index 5bb2da78fa2..ecf6ce48fde 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx @@ -11,7 +11,12 @@ import { SortingState, useReactTable, } from "@tanstack/react-table"; -import { getGuardrailLogoAndName, guardrail_provider_map, skipSystemMessageToChoice } from "./guardrail_info_helpers"; +import { + getGuardrailLogoAndName, + guardrail_provider_map, + skipSystemMessageToChoice, + skipToolMessageToChoice, +} from "./guardrail_info_helpers"; import EditGuardrailForm from "./edit_guardrail_form"; import { Guardrail, GuardrailDefinitionLocation } from "./types"; @@ -304,6 +309,9 @@ const GuardrailTable: React.FC = ({ skip_system_message_choice: skipSystemMessageToChoice( selectedGuardrail.litellm_params?.skip_system_message_in_guardrail, ), + skip_tool_message_choice: skipToolMessageToChoice( + selectedGuardrail.litellm_params?.skip_tool_message_in_guardrail, + ), ...selectedGuardrail.guardrail_info, }} /> From b379f4f98b3bf35f1a67163fbbae98d643ada02d Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 7 May 2026 22:41:16 -0700 Subject: [PATCH 29/85] [Feature] UI - Logs: Add 'Last Minute' to time-range quick select Adds a Last Minute option as the first entry in the logs page time-range quick-select dropdown. Tests verify the UI passes a ~1-minute window (start_date / end_date) to uiSpendLogsCall when selected. --- .../src/components/view_logs/constants.ts | 1 + .../src/components/view_logs/index.test.tsx | 50 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/ui/litellm-dashboard/src/components/view_logs/constants.ts b/ui/litellm-dashboard/src/components/view_logs/constants.ts index 57155feae23..5b0b1d0fee3 100644 --- a/ui/litellm-dashboard/src/components/view_logs/constants.ts +++ b/ui/litellm-dashboard/src/components/view_logs/constants.ts @@ -19,6 +19,7 @@ export const MCP_CALL_TYPES = ["call_mcp_tool", "list_mcp_tools"]; export const AGENT_CALL_TYPES = ["asend_message"]; export const QUICK_SELECT_OPTIONS: { label: string; value: number; unit: string }[] = [ + { label: "Last Minute", value: 1, unit: "minutes" }, { label: "Last 15 Minutes", value: 15, unit: "minutes" }, { label: "Last Hour", value: 1, unit: "hours" }, { label: "Last 4 Hours", value: 4, unit: "hours" }, diff --git a/ui/litellm-dashboard/src/components/view_logs/index.test.tsx b/ui/litellm-dashboard/src/components/view_logs/index.test.tsx index 937844e1c1d..7a9a541d3e0 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.test.tsx @@ -1,10 +1,12 @@ import { render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; +import moment from "moment"; import { beforeEach, describe, expect, it, vi } from "vitest"; import SpendLogsTable, { RequestViewer } from "./index"; import type { LogEntry } from "./columns"; import type { Row } from "@tanstack/react-table"; import { renderWithProviders } from "../../../tests/test-utils"; +import { uiSpendLogsCall } from "../networking"; const mockHandleFilterResetFromHook = vi.fn(); vi.mock("./log_filter_logic", async (importOriginal) => { @@ -238,4 +240,52 @@ describe("SpendLogsTable", () => { expect(inputsAfterReset.length).toBe(0); }); }); + + describe("Quick Select time range", () => { + const waitForWindowSeconds = async (minMinutes: number) => { + let diff = -1; + await waitFor(() => { + const lastCall = vi.mocked(uiSpendLogsCall).mock.calls.at(-1)?.[0]; + if (!lastCall) throw new Error("uiSpendLogsCall was not called"); + diff = moment + .utc(lastCall.end_date, "YYYY-MM-DD HH:mm:ss") + .diff(moment.utc(lastCall.start_date, "YYYY-MM-DD HH:mm:ss"), "seconds"); + // start_date is rounded down to the minute boundary; end_date is current time + expect(diff).toBeGreaterThanOrEqual(minMinutes * 60); + expect(diff).toBeLessThan((minMinutes + 1) * 60); + }); + return diff; + }; + + it("should pass a ~1-minute window to uiSpendLogsCall when 'Last Minute' is selected", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: /Last 24 Hours/i })); + await user.click(await screen.findByRole("button", { name: "Last Minute" })); + + await waitForWindowSeconds(1); + }); + + it("should pass a ~15-minute window to uiSpendLogsCall when 'Last 15 Minutes' is selected", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: /Last 24 Hours/i })); + await user.click(await screen.findByRole("button", { name: "Last 15 Minutes" })); + + await waitForWindowSeconds(15); + }); + + it("should update the time-range button label to 'Last Minute' after selecting it", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: /Last 24 Hours/i })); + await user.click(await screen.findByRole("button", { name: "Last Minute" })); + + expect(screen.getByRole("button", { name: "Last Minute" })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /Last 24 Hours/i })).not.toBeInTheDocument(); + }); + }); }); From ae67cecc22b069be64f6e4f197b3e8e5b1cfeb13 Mon Sep 17 00:00:00 2001 From: oss-agent-shin Date: Fri, 8 May 2026 15:30:41 -0700 Subject: [PATCH 30/85] Allow team admins to test model connections (#27487) Co-authored-by: oss-agent-shin <279349115+oss-agent-shin@users.noreply.github.com> Co-authored-by: ishaan-berri --- litellm/proxy/_types.py | 2 ++ .../proxy/auth/test_route_checks.py | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ed20fe86cdc..1c380392349 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -707,6 +707,8 @@ class LiteLLMRoutes(enum.Enum): # Project read routes - endpoint scopes results to caller's teams (non-admin) "/project/list", "/project/info", + # Endpoint enforces proxy-admin vs team-admin model access itself. + "/health/test_connection", # Invitation routes - org/team admins checked in endpoint via _user_has_admin_privileges "/invitation/new", "/invitation/delete", diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 268cdff1f7d..9c1cd116e69 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -80,6 +80,28 @@ def test_compliance_routes_open_to_internal_user(route): ) +def test_health_test_connection_route_delegates_internal_user_auth_to_endpoint(): + """Team model test-connection requests are authorized by the endpoint.""" + role = LitellmUserRoles.INTERNAL_USER.value + user_obj = LiteLLM_UserTable( + user_id="test_user", + user_email="test@example.com", + user_role=role, + ) + valid_token = UserAPIKeyAuth(user_id="test_user", user_role=role) + request = MagicMock(spec=Request) + request.query_params = {} + + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=role, + route="/health/test_connection", + request=request, + valid_token=valid_token, + request_data={}, + ) + + @pytest.mark.parametrize( "route", ["/compliance/eu-ai-act", "/compliance/gdpr"], From f2e97380d2deeeb8ab4b465f6cf5ee91d94233a9 Mon Sep 17 00:00:00 2001 From: oss-agent-shin Date: Fri, 8 May 2026 16:25:45 -0700 Subject: [PATCH 31/85] Add OpenRouter Qwen 3.6 Plus metadata (#27486) Co-authored-by: oss-agent-shin <279349115+oss-agent-shin@users.noreply.github.com> Co-authored-by: ishaan-berri --- ...odel_prices_and_context_window_backup.json | 14 ++++++++++++++ model_prices_and_context_window.json | 14 ++++++++++++++ tests/test_litellm/test_cost_calculator.py | 19 +++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4fba1980103..76d2d35a53a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -27187,6 +27187,20 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/qwen/qwen3.6-plus": { + "input_cost_per_token": 3.25e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.95e-06, + "source": "https://openrouter.ai/qwen/qwen3.6-plus", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "openrouter/qwen/qwen3.5-35b-a3b": { "input_cost_per_token": 2.5e-07, "litellm_provider": "openrouter", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 92e87c00ef6..e66ad8e0cf6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -27192,6 +27192,20 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/qwen/qwen3.6-plus": { + "input_cost_per_token": 3.25e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.95e-06, + "source": "https://openrouter.ai/qwen/qwen3.6-plus", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "openrouter/qwen/qwen3.5-35b-a3b": { "input_cost_per_token": 2.5e-07, "litellm_provider": "openrouter", diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index ebe175b2503..e53484dd287 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -110,6 +110,25 @@ def test_wandb_model_api_pricing_entries(): assert model_info["output_cost_per_token"] == output_cost +def test_openrouter_qwen36_plus_model_info(): + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model_info = litellm.model_cost.get("openrouter/qwen/qwen3.6-plus") + + assert model_info is not None + assert model_info["litellm_provider"] == "openrouter" + assert model_info["mode"] == "chat" + assert model_info["max_input_tokens"] == 1000000 + assert model_info["max_output_tokens"] == 65536 + assert model_info["input_cost_per_token"] == 3.25e-07 + assert model_info["output_cost_per_token"] == 1.95e-06 + assert model_info["supports_function_calling"] is True + assert model_info["supports_tool_choice"] is True + assert model_info["supports_reasoning"] is True + assert model_info["supports_vision"] is True + + def test_cost_calculator_with_usage(monkeypatch): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") From 13a193367f0ae2e7e8b29913e0d23a32d3bb0457 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 8 May 2026 17:39:39 -0700 Subject: [PATCH 32/85] feat(sso): show full IdP claims in /sso/debug/callback (#27498) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(sso): show full IdP claims in /sso/debug/callback The debug callback only displayed the proxy-parsed OpenID summary, so customers couldn't verify what custom claims (team_id, team_alias, roles, etc.) the IdP was actually returning. Render two new sections — Raw Claims (userinfo) and Access Token Claims (decoded JWT) — alongside the existing parsed view. Strip bearer tokens defense-in-depth in case a non-conforming IdP places them in its userinfo response. Resolves LIT-2838 * Update litellm/proxy/management_endpoints/ui_sso.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * fix(sso): hoist json.dumps out of f-string for py3.10 ruff --------- Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../html_forms/jwt_display_template.py | 203 ++++++++++++------ litellm/proxy/management_endpoints/ui_sso.py | 42 +++- .../proxy/management_endpoints/test_ui_sso.py | 160 ++++++++++++++ 3 files changed, 330 insertions(+), 75 deletions(-) diff --git a/litellm/proxy/common_utils/html_forms/jwt_display_template.py b/litellm/proxy/common_utils/html_forms/jwt_display_template.py index 03dff78dba0..ea65fd0e28f 100644 --- a/litellm/proxy/common_utils/html_forms/jwt_display_template.py +++ b/litellm/proxy/common_utils/html_forms/jwt_display_template.py @@ -14,7 +14,7 @@ jwt_display_template = """ padding: 20px; display: flex; justify-content: center; - align-items: center; + align-items: flex-start; min-height: 100vh; color: #333; } @@ -27,18 +27,18 @@ jwt_display_template = """ width: 800px; max-width: 100%; } - + .logo-container { text-align: center; margin-bottom: 30px; } - + .logo { font-size: 24px; font-weight: 600; color: #1e293b; } - + h2 { margin: 0 0 10px; color: #1e293b; @@ -46,7 +46,14 @@ jwt_display_template = """ font-weight: 600; text-align: center; } - + + h3 { + margin: 0 0 12px; + color: #1e293b; + font-size: 18px; + font-weight: 600; + } + .subtitle { color: #64748b; margin: 0 0 20px; @@ -58,15 +65,15 @@ jwt_display_template = """ background-color: #f1f5f9; border-radius: 6px; padding: 20px; - margin-bottom: 30px; + margin-bottom: 20px; border-left: 4px solid #2563eb; } - + .success-box { background-color: #f0fdf4; border-radius: 6px; padding: 20px; - margin-bottom: 30px; + margin-bottom: 20px; border-left: 4px solid #16a34a; } @@ -78,7 +85,7 @@ jwt_display_template = """ font-weight: 600; font-size: 16px; } - + .success-header { display: flex; align-items: center; @@ -87,46 +94,53 @@ jwt_display_template = """ font-weight: 600; font-size: 16px; } - + .info-header svg, .success-header svg { margin-right: 8px; } - + .data-container { margin-top: 20px; } - + .data-row { display: flex; border-bottom: 1px solid #e2e8f0; padding: 12px 0; } - + .data-row:last-child { border-bottom: none; } - + .data-label { font-weight: 500; color: #334155; - width: 180px; + width: 220px; flex-shrink: 0; } - + .data-value { color: #475569; word-break: break-all; } - + + .empty-note { + color: #64748b; + font-style: italic; + margin: 0; + font-size: 14px; + } + .jwt-container { background-color: #f8fafc; border-radius: 6px; padding: 15px; - margin-top: 20px; + margin-top: 12px; overflow-x: auto; border: 1px solid #e2e8f0; } - + .jwt-text { font-family: monospace; white-space: pre-wrap; @@ -134,7 +148,7 @@ jwt_display_template = """ margin: 0; color: #334155; } - + .back-button { display: inline-block; background-color: #6466E9; @@ -146,18 +160,18 @@ jwt_display_template = """ margin-top: 20px; text-align: center; } - + .back-button:hover { background-color: #4138C2; text-decoration: none; } - + .buttons { display: flex; gap: 10px; - margin-top: 20px; + margin-top: 12px; } - + .copy-button { background-color: #e2e8f0; color: #334155; @@ -169,11 +183,11 @@ jwt_display_template = """ display: flex; align-items: center; } - + .copy-button:hover { background-color: #cbd5e1; } - + .copy-button svg { margin-right: 6px; } @@ -188,7 +202,7 @@ jwt_display_template = """

SSO Debug Information

Results from the SSO authentication process.

- +
@@ -199,11 +213,7 @@ jwt_display_template = """

The SSO authentication completed successfully. Below is the information returned by the provider.

- -
- -
- +
@@ -211,22 +221,62 @@ jwt_display_template = """ - JSON Representation + Parsed by Proxy
+

Fields the proxy extracted into its internal user model.

+
+ +
+
+ +
+
+ + + + + + Raw Claims (userinfo) +
+

Complete set of claims returned by the IdP's userinfo endpoint.

- 
Loading...
+ 
Loading...
-
- + +
+
+ + + + + + Access Token Claims +
+

Decoded payload of the access token JWT (when the IdP issues one).

+
+ 
Loading...
+
+
+ +
+
+ Try Another SSO Login @@ -234,39 +284,58 @@ jwt_display_template = """ diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 74ee7c7220d..b13118db6fe 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -4078,6 +4078,8 @@ async def debug_sso_callback(request: Request): redirect_url += "/sso/debug/callback" result = None + received_response: Optional[dict] = None + access_token_payload: Optional[dict] = None if google_client_id is not None: result = await GoogleSSOHandler.get_google_callback_response( request=request, @@ -4094,12 +4096,14 @@ async def debug_sso_callback(request: Request): ) elif generic_client_id is not None: - result, _, _ = await get_generic_sso_response( - request=request, - jwt_handler=jwt_handler, - generic_client_id=generic_client_id, - redirect_url=redirect_url, - sso_jwt_handler=sso_jwt_handler, + result, received_response, access_token_payload = ( + await get_generic_sso_response( + request=request, + jwt_handler=jwt_handler, + generic_client_id=generic_client_id, + redirect_url=redirect_url, + sso_jwt_handler=sso_jwt_handler, + ) ) # If result is None, return a basic error message @@ -4128,10 +4132,32 @@ async def debug_sso_callback(request: Request): except Exception as e: filtered_result[key] = f"Complex value (not displayable): {str(e)}" + # Defense-in-depth: ensure no bearer tokens leak into the rendered HTML even if + # a non-conforming IdP places them in its userinfo response. + safe_raw_claims = { + k: v + for k, v in (received_response or {}).items() + if k not in _OAUTH_TOKEN_FIELDS + } + safe_access_token_claims = { + k: v + for k, v in (access_token_payload or {}).items() + if k not in _OAUTH_TOKEN_FIELDS + } + + sso_payload = { + "parsed_by_proxy": filtered_result, + "raw_claims": safe_raw_claims, + "access_token_claims": safe_access_token_claims, + } + # Replace the placeholder in the template with the actual data + sso_payload_json = json.dumps(sso_payload, indent=2, default=str).replace( + " Date: Fri, 8 May 2026 18:09:14 -0700 Subject: [PATCH 33/85] fix(proxy): point /metrics 401 at the opt-out flag Operators upgrading past 35bbca60b0 (which made /metrics auth default-on) see "Malformed API Key passed in. Ensure Key has 'Bearer ' prefix." with no hint that litellm_settings.require_auth_for_metrics_endpoint: false restores the previous unauthenticated behavior. Append that discovery hint to the existing 401 body so a Prometheus scraper that breaks after upgrade has a clear migration path. No behavior change. --- .../middleware/prometheus_auth_middleware.py | 5 ++++- .../test_prometheus_auth_middleware.py | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/middleware/prometheus_auth_middleware.py b/litellm/proxy/middleware/prometheus_auth_middleware.py index cfc4cbd64b2..7eb8ae83cb4 100644 --- a/litellm/proxy/middleware/prometheus_auth_middleware.py +++ b/litellm/proxy/middleware/prometheus_auth_middleware.py @@ -79,7 +79,10 @@ class PrometheusAuthMiddleware: # Send 401 response directly via ASGI protocol error_message = getattr(e, "message", str(e)) body = json.dumps( - f"Unauthorized access to metrics endpoint: {error_message}" + f"Unauthorized access to metrics endpoint: {error_message} " + f"To allow unauthenticated access, set " + f"`litellm_settings.require_auth_for_metrics_endpoint: false` " + f"in your proxy_config.yaml." ).encode("utf-8") await send( { diff --git a/tests/test_litellm/proxy/middleware/test_prometheus_auth_middleware.py b/tests/test_litellm/proxy/middleware/test_prometheus_auth_middleware.py index 6cab5baee9a..1d0c0f90fd1 100644 --- a/tests/test_litellm/proxy/middleware/test_prometheus_auth_middleware.py +++ b/tests/test_litellm/proxy/middleware/test_prometheus_auth_middleware.py @@ -121,6 +121,26 @@ def test_invalid_auth_metrics(app_with_middleware, monkeypatch): assert "Unauthorized access to metrics endpoint" in response.text +def test_invalid_auth_metrics_includes_optout_hint(app_with_middleware, monkeypatch): + """ + The 401 body must tell operators how to restore the previous unauthenticated + behavior, otherwise a Prometheus scraper that worked pre-upgrade just sees + "Malformed API Key" with no actionable migration path. + """ + monkeypatch.setattr(litellm, "require_auth_for_metrics_endpoint", True) + monkeypatch.setattr( + "litellm.proxy.middleware.prometheus_auth_middleware.user_api_key_auth", + fake_invalid_auth, + ) + + client = TestClient(app_with_middleware) + response = client.get("/metrics") + + assert response.status_code == 401, response.text + assert "require_auth_for_metrics_endpoint" in response.text + assert "false" in response.text + + def test_metrics_auth_uses_real_auth_when_route_is_public( app_with_middleware, monkeypatch ): From adc41ade8c0eb15ec6165906b2c3c9803b26d386 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 8 May 2026 20:18:31 -0700 Subject: [PATCH 34/85] fix(proxy): bound budget reservation per request instead of pinning to remaining headroom reserve_budget_for_request fell back to reserving the entire remaining team/key/user headroom whenever a request omitted max_tokens, which pinned the spend counter at max_budget for the duration of the in-flight request and false-positive-blocked every concurrent or back-to-back request until the success callback reconciled. Surfaced as an integration-test team being budget-blocked at its $2000 cap while DB spend was $0.144. Switch the missing-max_tokens path to a fixed default of 16384 output tokens (mirrors parallel_request_limiter_v3's DEFAULT_MAX_TOKENS_ESTIMATE precedent), and clamp explicit max_tokens at the model's max_output_tokens for reservation accounting only. The outbound request body is unchanged, so providers see whatever the caller actually sent; only the local integer used to compute reservation cost is bounded. This also prevents a hostile max_tokens=999999999 from inflating one request's reservation up to the entire team headroom. For Opus 4.7 (output $25/M, max_output 128K) on a $2000 budget the worst-case per-request reservation drops from "everything left" to $3.20, raising admittable concurrency from 1 to ~625. --- .../spend_tracking/budget_reservation.py | 63 ++-- .../proxy/test_budget_reservation.py | 327 ++++++++---------- 2 files changed, 164 insertions(+), 226 deletions(-) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 1d296611bfc..feae368d23f 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -95,11 +95,9 @@ async def reserve_budget_for_request( route=route, llm_router=llm_router, ) - if reservation_cost is None: - reservation_cost = await _get_smallest_remaining_budget( - counters=counters, - current_spend_by_counter_key=current_spend_by_counter_key, - ) + # estimate_request_max_cost still returns None when the model is unknown + # to the cost map (no token-priced cost fields, e.g. image/audio routes). + # In that case we fall back to read-time enforcement only. if reservation_cost is None or reservation_cost <= 0: return None @@ -553,32 +551,6 @@ def _coerce_window(window: Any) -> dict: return {} -async def _get_smallest_remaining_budget( - counters: List[_BudgetCounter], - current_spend_by_counter_key: Dict[str, float], -) -> Optional[float]: - remaining_budget: Optional[float] = None - for counter in counters: - current_spend = await _get_current_counter_value(counter=counter) - current_spend_by_counter_key[counter.counter_key] = current_spend - remaining = counter.max_budget - current_spend - if remaining <= 0: - raise litellm.BudgetExceededError( - current_cost=current_spend, - max_budget=counter.max_budget, - message=( - "Budget has been exceeded! " - f"{counter.entity_type}={counter.entity_id} " - f"Current cost: {current_spend}, " - f"Max budget: {counter.max_budget}" - ), - ) - remaining_budget = ( - remaining if remaining_budget is None else min(remaining_budget, remaining) - ) - return remaining_budget - - async def _reserve_counter( counter: _BudgetCounter, reservation_cost: float, @@ -946,6 +918,9 @@ def _estimate_input_tokens( return None +DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK = 16384 + + def _estimate_output_tokens( request_body: dict, route: str, @@ -954,15 +929,27 @@ def _estimate_output_tokens( if _is_input_only_route(route=route): return 0 + requested: Optional[int] = None for key in ("max_completion_tokens", "max_tokens", "max_output_tokens"): - max_tokens = _to_int(request_body.get(key)) - if max_tokens is not None: - return max_tokens + requested = _to_int(request_body.get(key)) + if requested is not None: + break - # If the caller did not cap output tokens, avoid reserving a model's - # theoretical maximum context. The caller can still admit one request by - # reserving the smallest remaining budget in reserve_budget_for_request(). - return None + # Clamp at min(requested-or-default, model_max-or-default). Two purposes: + # (1) Without an explicit cap we still need a finite reservation so the + # atomic admission counter actually bounds concurrent in-flight cost + # (mirrors parallel_request_limiter_v3's DEFAULT_MAX_TOKENS_ESTIMATE). + # (2) An adversarial caller cannot send max_tokens=999999999 to inflate + # the reservation up to remaining team headroom and pin the counter + # at the cap — the model can only physically emit max_output_tokens + # anyway, so reserving more is both wasteful and a DoS surface. + model_ceiling = ( + _to_int(model_info.get("max_output_tokens")) + or DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK + ) + if requested is None: + requested = DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK + return min(requested, model_ceiling) def _count_text_tokens(model: str, text: Any) -> int: diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 070b232066a..0e2ca98a113 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -585,15 +585,24 @@ async def test_should_cap_known_estimate_to_remaining_budget( @pytest.mark.asyncio -async def test_should_reserve_remaining_budget_when_output_cap_missing( +async def test_should_clamp_reservation_to_default_when_output_cap_missing( spend_counter_state, ): + """When max_tokens is not specified, _estimate_output_tokens falls back to + DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK (16K), clamped by the model's + max_output_tokens. Reservation must be a bounded per-request amount + (mirroring parallel_request_limiter_v3's DEFAULT_MAX_TOKENS_ESTIMATE), + not the entire remaining headroom.""" + from litellm.proxy.spend_tracking.budget_reservation import ( + DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK, + ) + counter_cache, key_cache = spend_counter_state proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) valid_token = UserAPIKeyAuth( token="key-budget-uncapped", spend=0.2, - max_budget=1.0, + max_budget=10000.0, ) await key_cache.async_set_cache( key="key-budget-uncapped", @@ -602,22 +611,24 @@ async def test_should_reserve_remaining_budget_when_output_cap_missing( request_body = _request_body() request_body.pop("max_tokens") + output_cost_per_token = 1e-5 # roughly Opus 4.5/4.7 output rate + expected_cost = DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK * output_cost_per_token + with patch( "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", return_value={ "input_cost_per_token": 0.0, - "output_cost_per_token": 100.0, - "max_output_tokens": 200000, + "output_cost_per_token": output_cost_per_token, + "max_output_tokens": 200000, # well above the 16K fallback }, ): - assert ( - estimate_request_max_cost( - request_body=request_body, - route="/chat/completions", - llm_router=None, - ) - is None + estimated = estimate_request_max_cost( + request_body=request_body, + route="/chat/completions", + llm_router=None, ) + assert estimated == pytest.approx(expected_cost) + reservation = await reserve_budget_for_request( request_body=request_body, route="/chat/completions", @@ -631,47 +642,45 @@ async def test_should_reserve_remaining_budget_when_output_cap_missing( ) assert reservation is not None - assert reservation["reserved_cost"] == pytest.approx(0.8) - assert counter_cache.in_memory_cache.get_cache( - key="spend:key:key-budget-uncapped" - ) == pytest.approx(1.0) - + assert reservation["reserved_cost"] == pytest.approx(expected_cost) await release_budget_reservation(reservation) @pytest.mark.asyncio -async def test_should_shrink_uncapped_reservation_when_counter_advances( +async def test_should_clamp_reservation_to_model_ceiling_when_caller_overrequests( spend_counter_state, - monkeypatch, ): + """An adversarial caller sending max_tokens=999_999_999 must not be able + to inflate the per-request reservation up to the entire remaining team + headroom. _estimate_output_tokens clamps the explicit value at the + model's max_output_tokens — the model can only physically emit that + many tokens anyway, so anything more is both wasteful and a DoS surface.""" counter_cache, key_cache = spend_counter_state proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) valid_token = UserAPIKeyAuth( - token="key-budget-uncapped-race", - spend=0.2, - max_budget=1.0, + token="key-budget-overrequest", + spend=0.0, + max_budget=10000.0, ) + await key_cache.async_set_cache( + key="key-budget-overrequest", + value=valid_token, + ) + request_body = _request_body() - request_body.pop("max_tokens") + request_body["max_tokens"] = 999_999_999 - from litellm.proxy.spend_tracking import budget_reservation - - async def stale_counter_read(counter): - await counter_cache.async_increment_cache( - key=counter.counter_key, - value=0.3, - ) - return 0.2 - - monkeypatch.setattr( - budget_reservation, - "_get_current_counter_value", - stale_counter_read, - ) + output_cost_per_token = 1e-5 + model_ceiling = 128_000 + expected_cost = model_ceiling * output_cost_per_token with patch( - "litellm.proxy.spend_tracking.budget_reservation.estimate_request_max_cost", - return_value=None, + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "input_cost_per_token": 0.0, + "output_cost_per_token": output_cost_per_token, + "max_output_tokens": model_ceiling, + }, ): reservation = await reserve_budget_for_request( request_body=request_body, @@ -686,101 +695,9 @@ async def test_should_shrink_uncapped_reservation_when_counter_advances( ) assert reservation is not None - assert reservation["reserved_cost"] == pytest.approx(0.7) - assert counter_cache.in_memory_cache.get_cache( - key="spend:key:key-budget-uncapped-race" - ) == pytest.approx(1.0) - + assert reservation["reserved_cost"] == pytest.approx(expected_cost) await release_budget_reservation(reservation) - assert counter_cache.in_memory_cache.get_cache( - key="spend:key:key-budget-uncapped-race" - ) == pytest.approx(0.3) - - -@pytest.mark.asyncio -async def test_should_shrink_uncapped_reservation_multiple_times( - spend_counter_state, - monkeypatch, -): - counter_cache, key_cache = spend_counter_state - proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) - valid_token = UserAPIKeyAuth( - token="key-budget-double-resize", - spend=0.2, - max_budget=1.0, - team_id="team-budget-double-resize", - ) - team_object = LiteLLM_TeamTable( - team_id="team-budget-double-resize", - spend=0.2, - max_budget=1.0, - ) - request_body = _request_body() - request_body.pop("max_tokens") - - from litellm.proxy.spend_tracking import budget_reservation - - stale_spend_by_counter_key = { - "spend:key:key-budget-double-resize": 0.3, - "spend:team:team-budget-double-resize": 0.4, - } - - async def stale_counter_read(counter): - await counter_cache.async_increment_cache( - key=counter.counter_key, - value=stale_spend_by_counter_key[counter.counter_key], - ) - return 0.2 - - monkeypatch.setattr( - budget_reservation, - "_get_current_counter_value", - stale_counter_read, - ) - - with patch( - "litellm.proxy.spend_tracking.budget_reservation.estimate_request_max_cost", - return_value=None, - ): - reservation = await reserve_budget_for_request( - request_body=request_body, - route="/chat/completions", - llm_router=None, - valid_token=valid_token, - team_object=team_object, - user_object=None, - prisma_client=None, - user_api_key_cache=key_cache, - proxy_logging_obj=proxy_logging_obj, - ) - - assert reservation is not None - assert reservation["reserved_cost"] == pytest.approx(0.6) - assert [entry["reserved_cost"] for entry in reservation["entries"]] == [ - pytest.approx(0.6), - pytest.approx(0.6), - ] - assert [entry["applied_adjustment"] for entry in reservation["entries"]] == [ - pytest.approx(0.0), - pytest.approx(0.0), - ] - assert counter_cache.in_memory_cache.get_cache( - key="spend:key:key-budget-double-resize" - ) == pytest.approx(0.9) - assert counter_cache.in_memory_cache.get_cache( - key="spend:team:team-budget-double-resize" - ) == pytest.approx(1.0) - - await release_budget_reservation(reservation) - - assert counter_cache.in_memory_cache.get_cache( - key="spend:key:key-budget-double-resize" - ) == pytest.approx(0.3) - assert counter_cache.in_memory_cache.get_cache( - key="spend:team:team-budget-double-resize" - ) == pytest.approx(0.4) - def test_should_start_window_without_reset_at_at_duration_boundary(): before = datetime.now(timezone.utc) - timedelta(hours=1) @@ -1047,62 +964,6 @@ async def test_should_release_tracked_entry_when_reservation_fails_after_increme ) == pytest.approx(0.0) -@pytest.mark.asyncio -async def test_should_not_re_read_uncapped_budget_after_reservation_fallback( - spend_counter_state, - monkeypatch, -): - _, key_cache = spend_counter_state - proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) - valid_token = UserAPIKeyAuth( - token="key-budget-uncapped-read-once", - spend=0.2, - max_budget=1.0, - ) - - from litellm.proxy.spend_tracking import budget_reservation - - current_counter_reads = [] - - async def mock_get_current_counter_value(counter): - current_counter_reads.append(counter.counter_key) - return counter.fallback_spend - - async def mock_reserve_counter(counter, reservation_cost): - return None - - monkeypatch.setattr( - budget_reservation, - "_get_current_counter_value", - mock_get_current_counter_value, - ) - monkeypatch.setattr( - budget_reservation, - "_reserve_counter", - mock_reserve_counter, - ) - - with patch( - "litellm.proxy.spend_tracking.budget_reservation.estimate_request_max_cost", - return_value=None, - ): - reservation = await reserve_budget_for_request( - request_body=_request_body(), - route="/chat/completions", - llm_router=None, - valid_token=valid_token, - team_object=None, - user_object=None, - prisma_client=None, - user_api_key_cache=key_cache, - proxy_logging_obj=proxy_logging_obj, - ) - - assert reservation is not None - assert reservation["reserved_cost"] == pytest.approx(0.8) - assert current_counter_reads == ["spend:key:key-budget-uncapped-read-once"] - - @pytest.mark.asyncio async def test_should_reconcile_reserved_counter_to_actual_spend( spend_counter_state, @@ -1492,4 +1353,94 @@ async def test_should_reserve_all_budgeted_counters(spend_counter_state): counter_cache.in_memory_cache.get_cache(key="spend:team:team-budget-all") == 0.3 ) - await release_budget_reservation(reservation) + +@pytest.mark.asyncio +async def test_should_not_block_concurrent_team_request_when_first_request_lacks_max_tokens( + spend_counter_state, +): + """ + Regression test: a team-bound request with no max_tokens must not pin the + team's spend counter at max_budget for the duration of the request. + + Repro of the integration-test team being falsely budget-blocked at the + $2000 cap while DB spend is $0.144: the first request without max_tokens + used to reserve the entire remaining headroom, leaving any subsequent + request stuck behind a counter sitting at the cap until the success + callback finished reconciling. + """ + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + + valid_token = UserAPIKeyAuth( + token="key-team-integration-tests", + spend=0.0, + team_id="team-integration-tests", + ) + team_object = LiteLLM_TeamTable( + team_id="team-integration-tests", + max_budget=2000.0, + spend=0.144, + ) + await key_cache.async_set_cache( + key=f"team_id:{team_object.team_id}", + value=team_object, + ) + + request_body = _request_body() + request_body.pop("max_tokens") + + # Realistic Opus 4.7 output pricing — the 16K fallback × $25/M ≈ $0.40 + # reservation per request, leaving ~5000 admittable concurrent requests + # against a $2000 team budget. + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "input_cost_per_token": 5e-6, + "output_cost_per_token": 2.5e-5, + "max_output_tokens": 128000, + }, + ): + first_reservation = await reserve_budget_for_request( + request_body=request_body, + route="/chat/completions", + llm_router=None, + valid_token=valid_token, + team_object=team_object, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + # The team counter must not be pinned at max_budget while the first + # request is in flight, otherwise concurrent requests false-positive. + team_counter_after_first = ( + counter_cache.in_memory_cache.get_cache( + key=f"spend:team:{team_object.team_id}" + ) + or 0.0 + ) + assert team_counter_after_first < team_object.max_budget, ( + f"Team counter sat at {team_counter_after_first} after one uncapped " + f"reservation against a {team_object.max_budget} budget — concurrent " + "requests will be falsely blocked." + ) + + # Second request — same shape — must succeed without raising. + second_reservation = await reserve_budget_for_request( + request_body=request_body, + route="/chat/completions", + llm_router=None, + valid_token=valid_token, + team_object=team_object, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + assert second_reservation is not None + + if first_reservation is not None: + await release_budget_reservation(first_reservation) + if second_reservation is not None: + await release_budget_reservation(second_reservation) From b5d3a5fc856ed1cf9b101d37bd0ec6d6d44751b2 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Fri, 8 May 2026 21:05:50 -0700 Subject: [PATCH 35/85] feat: add read-replica routing for Prisma DB via DATABASE_URL_READ_REPLICA (#27493) - Introduce RoutingPrismaWrapper that transparently routes read operations (find_*, count, group_by, query_raw, query_first) to a reader endpoint while writes remain on the writer, enabling Aurora-style reader/writer endpoint splits - Add IAMEndpoint dataclass and parse_iam_endpoint_from_url() to capture static connection fields from a reader URL so only the IAM token needs to rotate, avoiding the need for separate DATABASE_HOST_READ_REPLICA/etc. env vars - Enhance PrismaWrapper with per-instance knobs (db_url_env_var, iam_endpoint, recreate_uses_datasource, log_prefix) so writer and reader wrappers are independent: the reader writes its fresh URL to DATABASE_URL_READ_REPLICA and passes datasource override to Prisma since Prisma only auto-reads DATABASE_URL - Fix deadlock in PrismaWrapper.__getattr__: when called from inside a running event loop, schedule the token refresh as a background task instead of blocking with run_coroutine_threadsafe + future.result(), which would deadlock the loop thread waiting for a coroutine that needs the loop to run - Fix botocore crash when DATABASE_PORT is unset by defaulting to "5432" in both proxy_cli.py and PrismaWrapper.get_rds_iam_token(); passing None caused botocore to embed the literal string "None" in the presigned URL - Implement graceful reader degradation: reader connect/recreate failures are non-fatal; wrapper sets _reader_unavailable=True and silently routes reads to the writer to keep the proxy serving traffic during transient reader outages - Add PrismaClient.writer_db property so the reconnect smoke-test always validates the writer engine specifically; query_raw on the routing wrapper would route to the reader and not verify the newly-recreated writer - Expose DATABASE_URL_READ_REPLICA in Helm chart (values.yaml + deployment.yaml) via both plain value and secret key reference, and document the field in docker-compose.yml - Add 887-line test suite covering routing logic, IAM token refresh paths, reader degradation scenarios, datasource override behavior, and the deadlock regression Co-authored-by: Yassin Kortam --- .../litellm-helm/templates/deployment.yaml | 10 + deploy/charts/litellm-helm/values.yaml | 20 + docker-compose.yml | 5 + litellm/proxy/db/prisma_client.py | 237 +++-- litellm/proxy/db/routing_prisma_wrapper.py | 213 +++++ litellm/proxy/proxy_cli.py | 7 +- litellm/proxy/utils.py | 121 ++- .../proxy/db/test_routing_prisma_wrapper.py | 887 ++++++++++++++++++ 8 files changed, 1423 insertions(+), 77 deletions(-) create mode 100644 litellm/proxy/db/routing_prisma_wrapper.py create mode 100644 tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 6aa1771b7bb..25f69080878 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -100,6 +100,16 @@ spec: - name: DATABASE_URL value: {{ .Values.db.url | quote }} {{- end }} + {{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }} + - name: DATABASE_URL_READ_REPLICA + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.readReplicaUrlKey }} + {{- else if .Values.db.readReplicaUrl }} + - name: DATABASE_URL_READ_REPLICA + value: {{ .Values.db.readReplicaUrl | quote }} + {{- end }} - name: PROXY_MASTER_KEY valueFrom: secretKeyRef: diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index ba4059e0840..9c7c013341b 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -252,6 +252,26 @@ db: passwordKey: password # Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint endpointKey: "" + # Optional: when set, DATABASE_URL_READ_REPLICA will be sourced from this + # secret key instead of db.readReplicaUrl. Prefer this over the plain + # value: read-replica URLs typically embed credentials, and a value + # written to db.readReplicaUrl ends up visible in the rendered pod spec + # and the Helm release secret. + readReplicaUrlKey: "" + + # Optional read-replica routing. When set, the proxy sends read-only + # queries (find_*, count, group_by, query_raw/_first) to this URL while + # writes continue to go to db.url. Useful for Aurora-style clusters with + # separate reader/writer endpoints. Leave empty to keep single-DB behavior. + # When IAM_TOKEN_DB_AUTH is enabled, the reader URL is auto-refreshed + # alongside the writer (host/port/user/db are parsed from this URL once + # at startup; only the IAM token rotates). + # + # If the URL embeds credentials, prefer db.secret.readReplicaUrlKey over + # this field — the plain value is rendered into the pod spec and the + # Helm release secret. This field is intended for credential-less URLs + # only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime). + readReplicaUrl: "" # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster. # The Stackgres Operator must already be installed within the target diff --git a/docker-compose.yml b/docker-compose.yml index 988860a7877..80e1f289aad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,11 @@ services: - "4000:4000" # Map the container port to the host, change the host port if necessary environment: DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm" + # Optional: route read-only queries (find_*, count, group_by, query_raw/_first) + # to a separate reader endpoint, e.g. an Aurora reader. Leave unset for + # single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL + # is auto-refreshed alongside the writer. + # DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm" STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI env_file: - .env # Load local .env file diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index d112e222307..af5a58802bb 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -10,13 +10,64 @@ import subprocess import time import urllib import urllib.parse +from dataclasses import dataclass from datetime import datetime, timedelta -from typing import Any, Optional, Union +from typing import Any, Dict, Optional, Union from litellm._logging import verbose_proxy_logger from litellm.secret_managers.main import str_to_bool +@dataclass(frozen=True) +class IAMEndpoint: + """Static parts of an RDS IAM-authenticated Postgres connection. + + The IAM token rotates every ~15 minutes; everything else (host, port, user, + database name, schema) stays fixed. We capture the static fields once so + refresh just regenerates the token and reassembles the URL. + """ + + host: str + port: str + user: str + name: str + schema: Optional[str] = None + + def build_url(self, token: str) -> str: + url = f"postgresql://{self.user}:{token}@{self.host}:{self.port}/{self.name}" + if self.schema: + url += f"?schema={self.schema}" + return url + + +def parse_iam_endpoint_from_url(url: str) -> IAMEndpoint: + """Parse an IAMEndpoint from a Postgres URL. + + Used so a reader URL can drive its own IAM refresh without requiring + callers to set parallel DATABASE_HOST_READ_REPLICA / etc. env vars. + """ + parsed = urllib.parse.urlparse(url) + if not parsed.hostname or not parsed.username: + raise ValueError("Cannot parse IAM endpoint from URL: missing host or username") + name = (parsed.path or "/").lstrip("/") + if not name: + raise ValueError("Cannot parse IAM endpoint from URL: missing database name") + port = str(parsed.port) if parsed.port else "5432" + schema: Optional[str] = None + if parsed.query: + qs = urllib.parse.parse_qs(parsed.query) + schema_vals = qs.get("schema") + if schema_vals: + schema = schema_vals[0] + return IAMEndpoint( + host=parsed.hostname, + port=port, + user=parsed.username, + name=name, + schema=schema, + ) + + class PrismaWrapper: """ Wrapper around Prisma client that handles RDS IAM token authentication. @@ -37,10 +88,33 @@ class PrismaWrapper: # Fallback refresh interval if token parsing fails (10 minutes) FALLBACK_REFRESH_INTERVAL_SECONDS = 600 - def __init__(self, original_prisma: Any, iam_token_db_auth: bool): + def __init__( + self, + original_prisma: Any, + iam_token_db_auth: bool, + *, + db_url_env_var: str = "DATABASE_URL", + iam_endpoint: Optional[IAMEndpoint] = None, + recreate_uses_datasource: bool = False, + log_prefix: str = "", + ): self._original_prisma = original_prisma self.iam_token_db_auth = iam_token_db_auth + # Per-connection knobs so the same wrapper can be used for the writer + # (defaults: DATABASE_URL env, IAM endpoint from DATABASE_HOST/etc., + # recreate via env reload) or for a reader (DATABASE_URL_READ_REPLICA + # env, IAM endpoint parsed from that URL, recreate via datasource + # override since Prisma only auto-reads DATABASE_URL). + self._db_url_env_var = db_url_env_var + self._iam_endpoint = iam_endpoint + self._recreate_uses_datasource = recreate_uses_datasource + # Tag every log line emitted by this wrapper instance so writer and + # reader can be told apart in interleaved output (e.g. "[writer] RDS + # IAM token refresh scheduled in 720 seconds"). Empty string (default) + # keeps backward-compatible logs for the single-DB case. + self._log_prefix = f"{log_prefix} " if log_prefix else "" + # Background token refresh task management self._token_refresh_task: Optional[asyncio.Task] = None self._reconnection_lock = asyncio.Lock() @@ -157,7 +231,7 @@ class PrismaWrapper: Returns 0 if token should be refreshed immediately. Returns FALLBACK_REFRESH_INTERVAL_SECONDS if parsing fails. """ - db_url = os.getenv("DATABASE_URL") + db_url = os.getenv(self._db_url_env_var) token = self._extract_token_from_db_url(db_url) expiration_time = self._parse_token_expiration(token) @@ -199,12 +273,30 @@ class PrismaWrapper: return datetime.utcnow() > expiration_time def get_rds_iam_token(self) -> Optional[str]: - """Generate a new RDS IAM token and update DATABASE_URL.""" - if self.iam_token_db_auth: - from litellm.proxy.auth.rds_iam_token import generate_iam_auth_token + """Generate a new RDS IAM token and update the configured DB URL env var. + When the wrapper was constructed with an explicit `iam_endpoint` + (typical for a reader wrapper whose host/port/user came from a parsed + URL), use that. Otherwise fall back to the legacy DATABASE_HOST/PORT/ + USER/NAME/SCHEMA env vars (writer behavior). + """ + if not self.iam_token_db_auth: + return None + + from litellm.proxy.auth.rds_iam_token import generate_iam_auth_token + + if self._iam_endpoint is not None: + endpoint = self._iam_endpoint + token = generate_iam_auth_token( + db_host=endpoint.host, db_port=endpoint.port, db_user=endpoint.user + ) + _db_url = endpoint.build_url(token) + else: db_host = os.getenv("DATABASE_HOST") - db_port = os.getenv("DATABASE_PORT") + # Default to the Postgres standard port; passing None to + # `generate_iam_auth_token` makes botocore embed the literal + # string "None" in the presigned URL, which then fails to parse. + db_port = os.getenv("DATABASE_PORT", "5432") db_user = os.getenv("DATABASE_USER") db_name = os.getenv("DATABASE_NAME") db_schema = os.getenv("DATABASE_SCHEMA") @@ -217,9 +309,8 @@ class PrismaWrapper: if db_schema: _db_url += f"?schema={db_schema}" - os.environ["DATABASE_URL"] = _db_url - return _db_url - return None + os.environ[self._db_url_env_var] = _db_url + return _db_url async def recreate_prisma_client( self, new_db_url: str, http_client: Optional[Any] = None @@ -231,6 +322,11 @@ class PrismaWrapper: synchronous `subprocess.Popen.wait()` that can freeze the asyncio event loop for 30-120+ seconds when the engine is stuck on TCP close, breaking `/health/liveliness` and causing Kubernetes pod restarts. + + The writer wrapper relies on Prisma re-reading `DATABASE_URL` from env; + the reader wrapper opts into `recreate_uses_datasource=True` so the + new URL is passed explicitly via `datasource={"url": ...}` (Prisma + does not auto-read alternate env vars like DATABASE_URL_READ_REPLICA). """ from prisma import Prisma # type: ignore @@ -238,10 +334,12 @@ class PrismaWrapper: if old_engine_pid > 0: await self._kill_engine_process(old_engine_pid) + kwargs: Dict[str, Any] = {} if http_client is not None: - self._original_prisma = Prisma(http=http_client) - else: - self._original_prisma = Prisma() + kwargs["http"] = http_client + if self._recreate_uses_datasource: + kwargs["datasource"] = {"url": new_db_url} + self._original_prisma = Prisma(**kwargs) await self._original_prisma.connect() @@ -265,7 +363,8 @@ class PrismaWrapper: self._token_refresh_task = asyncio.create_task(self._token_refresh_loop()) verbose_proxy_logger.info( - "Started RDS IAM token proactive refresh background task" + "%sStarted RDS IAM token proactive refresh background task", + self._log_prefix, ) async def stop_token_refresh_task(self) -> None: @@ -283,7 +382,9 @@ class PrismaWrapper: except asyncio.CancelledError: pass self._token_refresh_task = None - verbose_proxy_logger.info("Stopped RDS IAM token refresh background task") + verbose_proxy_logger.info( + "%sStopped RDS IAM token refresh background task", self._log_prefix + ) async def _token_refresh_loop(self) -> None: """ @@ -294,7 +395,7 @@ class PrismaWrapper: This is more efficient than polling, requiring only 1 wake-up per token cycle. """ verbose_proxy_logger.info( - f"RDS IAM token refresh loop started. " + f"{self._log_prefix}RDS IAM token refresh loop started. " f"Tokens will be refreshed {self.TOKEN_REFRESH_BUFFER_SECONDS}s before expiration." ) @@ -305,21 +406,25 @@ class PrismaWrapper: if sleep_seconds > 0: verbose_proxy_logger.info( - f"RDS IAM token refresh scheduled in {sleep_seconds:.0f} seconds " - f"({sleep_seconds / 60:.1f} minutes)" + f"{self._log_prefix}RDS IAM token refresh scheduled in " + f"{sleep_seconds:.0f} seconds ({sleep_seconds / 60:.1f} minutes)" ) await asyncio.sleep(sleep_seconds) # Refresh the token - verbose_proxy_logger.info("Proactively refreshing RDS IAM token...") + verbose_proxy_logger.info( + "%sProactively refreshing RDS IAM token...", self._log_prefix + ) await self._safe_refresh_token() except asyncio.CancelledError: - verbose_proxy_logger.info("RDS IAM token refresh loop cancelled") + verbose_proxy_logger.info( + "%sRDS IAM token refresh loop cancelled", self._log_prefix + ) break except Exception as e: verbose_proxy_logger.error( - f"Error in RDS IAM token refresh loop: {e}. " + f"{self._log_prefix}Error in RDS IAM token refresh loop: {e}. " f"Retrying in {self.FALLBACK_REFRESH_INTERVAL_SECONDS}s..." ) # On error, wait before retrying to avoid tight error loops @@ -341,65 +446,75 @@ class PrismaWrapper: await self.recreate_prisma_client(new_db_url) self._last_refresh_time = datetime.utcnow() verbose_proxy_logger.info( - "RDS IAM token refreshed successfully. New token valid for ~15 minutes." + "%sRDS IAM token refreshed successfully. New token valid for ~15 minutes.", + self._log_prefix, ) else: verbose_proxy_logger.error( - "Failed to generate new RDS IAM token during proactive refresh" + "%sFailed to generate new RDS IAM token during proactive refresh", + self._log_prefix, ) def __getattr__(self, name: str): """ Proxy attribute access to the underlying Prisma client. - If IAM token auth is enabled and the token is expired, this method - provides a synchronous fallback to refresh the token. However, this - should rarely be needed since the background task proactively refreshes - tokens before they expire. + If IAM token auth is enabled and the token is found expired here, the + proactive refresh task has missed its window. Behavior depends on + whether we're called from inside a running event loop: - FIXED: Now properly waits for reconnection to complete before returning, - instead of the previous fire-and-forget pattern that caused the bug. + - Inside the loop (typical: from a coroutine): schedule a refresh as a + background task and return the (stale) attribute. The caller's await + will likely fail with a connection error and be retried by upper + layers (`call_with_db_reconnect_retry`); by that time the refresh + has either completed or escalated to the proactive loop's error + path. We CANNOT block here — `run_coroutine_threadsafe(...)` + + `future.result()` from inside the same loop deadlocks the loop + (loop thread is blocked, scheduled coroutine never runs, 30s timeout). + + - No running loop (sync caller, mostly tests): run the refresh in a + fresh loop and re-fetch the attribute. """ original_attr = getattr(self._original_prisma, name) if self.iam_token_db_auth: - db_url = os.getenv("DATABASE_URL") + db_url = os.getenv(self._db_url_env_var) # Check if token is expired (should be rare if background task is running) if self.is_token_expired(db_url): - verbose_proxy_logger.warning( - "RDS IAM token expired in __getattr__ - proactive refresh may have failed. " - "Triggering synchronous fallback refresh..." - ) + try: + running_loop = asyncio.get_running_loop() + except RuntimeError: + running_loop = None - new_db_url = self.get_rds_iam_token() - if new_db_url: - loop = asyncio.get_event_loop() - - if loop.is_running(): - # FIXED: Actually wait for the reconnection to complete! - # The previous code used fire-and-forget which caused the bug. - future = asyncio.run_coroutine_threadsafe( - self.recreate_prisma_client(new_db_url), loop - ) - try: - # Wait up to 30 seconds for reconnection - future.result(timeout=30) - verbose_proxy_logger.info( - "Synchronous token refresh completed successfully" - ) - except Exception as e: - verbose_proxy_logger.error( - f"Failed to refresh token synchronously: {e}" - ) - raise - else: - asyncio.run(self.recreate_prisma_client(new_db_url)) - - # Get the NEW attribute after reconnection - original_attr = getattr(self._original_prisma, name) + if running_loop is not None: + verbose_proxy_logger.warning( + "%sRDS IAM token expired in __getattr__ — proactive refresh " + "may have failed. Scheduling async refresh; the current " + "request may fail and be retried with the fresh token.", + self._log_prefix, + ) + # Non-blocking: schedule the locked refresh on the + # running loop. The reconnection lock inside + # `_safe_refresh_token` coalesces concurrent triggers. + running_loop.create_task(self._safe_refresh_token()) else: - raise ValueError("Failed to get RDS IAM token") + verbose_proxy_logger.warning( + "%sRDS IAM token expired in __getattr__ — proactive refresh " + "may have failed. Triggering synchronous fallback refresh...", + self._log_prefix, + ) + new_db_url = self.get_rds_iam_token() + if new_db_url: + asyncio.run(self.recreate_prisma_client(new_db_url)) + # Re-fetch attribute against the recreated Prisma instance. + original_attr = getattr(self._original_prisma, name) + verbose_proxy_logger.info( + "%sSynchronous token refresh completed successfully", + self._log_prefix, + ) + else: + raise ValueError("Failed to get RDS IAM token") return original_attr diff --git a/litellm/proxy/db/routing_prisma_wrapper.py b/litellm/proxy/db/routing_prisma_wrapper.py new file mode 100644 index 00000000000..0a976e9f1ea --- /dev/null +++ b/litellm/proxy/db/routing_prisma_wrapper.py @@ -0,0 +1,213 @@ +""" +RoutingPrismaWrapper: routes Prisma reads to a read-replica client and writes +to a writer client. Used when DATABASE_URL_READ_REPLICA is configured; +otherwise PrismaClient uses the writer-only PrismaWrapper directly. +""" + +import os +from typing import Any, Callable, Optional + +from litellm._logging import verbose_proxy_logger +from litellm.proxy.db.prisma_client import PrismaWrapper + +# Per-model action methods that read from the database. These are routed to +# the read replica when one is configured. +_MODEL_READ_METHODS = frozenset( + { + "find_first", + "find_first_or_raise", + "find_many", + "find_unique", + "find_unique_or_raise", + "count", + "group_by", + "query_first", + "query_raw", + } +) + +# Top-level Prisma client methods that read from the database. +_TOP_LEVEL_READ_METHODS = frozenset({"query_first", "query_raw"}) + + +class _RoutedActions: + """Per-model accessor that sends reads to the reader and writes to the writer. + + `should_use_reader` is consulted on every read dispatch so a mid-call flip + of the routing wrapper's reader-availability flag (e.g. after the reader + fails a recreate) is observed without re-fetching the actions accessor. + """ + + __slots__ = ("_writer_actions", "_reader_actions", "_should_use_reader") + + def __init__( + self, + writer_actions: Any, + reader_actions: Any, + should_use_reader: Callable[[], bool], + ): + self._writer_actions = writer_actions + self._reader_actions = reader_actions + self._should_use_reader = should_use_reader + + def __getattr__(self, name: str) -> Any: + if name in _MODEL_READ_METHODS and self._should_use_reader(): + return getattr(self._reader_actions, name) + return getattr(self._writer_actions, name) + + +class RoutingPrismaWrapper: + """ + Routes Prisma operations between a writer and a reader Prisma client. + + Reads (find_*, count, group_by, query_raw, query_first) go to the reader; + everything else (writes, transactions, raw execute) goes to the writer. + Lifecycle methods (connect, disconnect, IAM token refresh) act on both + clients so callers do not need to know about the split. When + IAM_TOKEN_DB_AUTH is enabled, both writer and reader refresh their tokens + independently on their own ~12-minute cadence. + + Reader degradation: a reader-side failure (failed connect, failed + recreate) is non-fatal — the wrapper sets `_reader_unavailable=True`, logs + a warning, and routes subsequent reads to the writer. The next successful + `connect()` or `recreate_prisma_client()` clears the flag. This keeps the + proxy serving traffic during transient reader outages instead of failing + startup or returning errors for read-heavy endpoints. + """ + + def __init__(self, writer: PrismaWrapper, reader: PrismaWrapper): + self._writer = writer + self._reader = reader + # When True, reads fall back to the writer. Flipped on by reader + # connect/recreate failures and flipped off on the next reader recovery. + self._reader_unavailable: bool = False + + @property + def writer(self) -> PrismaWrapper: + return self._writer + + @property + def reader(self) -> PrismaWrapper: + return self._reader + + @property + def reader_unavailable(self) -> bool: + return self._reader_unavailable + + def _should_use_reader(self) -> bool: + return not self._reader_unavailable + + async def connect(self, *args: Any, **kwargs: Any) -> None: + await self._writer.connect(*args, **kwargs) + verbose_proxy_logger.info("[writer] DB connected") + try: + await self._reader.connect(*args, **kwargs) + self._reader_unavailable = False + verbose_proxy_logger.info("[reader] DB connected") + except Exception as e: + # Degrade gracefully: the proxy keeps serving traffic with reads + # routed to the writer until the reader endpoint is reachable. + # Aborting startup here would tie proxy availability to an + # opt-in, best-effort reader endpoint. + self._reader_unavailable = True + verbose_proxy_logger.warning( + "Failed to connect to read replica DB: %s. " + "Falling back to the writer for reads until the reader is reachable.", + e, + ) + + async def disconnect(self, *args: Any, **kwargs: Any) -> None: + first_error: Optional[BaseException] = None + for client in (self._writer, self._reader): + try: + await client.disconnect(*args, **kwargs) + except Exception as e: + if first_error is None: + first_error = e + verbose_proxy_logger.warning("Error disconnecting Prisma client: %s", e) + if first_error is not None: + raise first_error + + def is_connected(self) -> bool: + # Reflects writer health only. The reader is best-effort; its + # availability is tracked via `_reader_unavailable` and a degraded + # reader must NOT cause a writer reconnect (would loop indefinitely + # since recreate_prisma_client only fixes writer-side problems). + return bool(self._writer.is_connected()) + + async def start_token_refresh_task(self) -> None: + await self._writer.start_token_refresh_task() + await self._reader.start_token_refresh_task() + + async def stop_token_refresh_task(self) -> None: + await self._writer.stop_token_refresh_task() + await self._reader.stop_token_refresh_task() + + async def recreate_prisma_client( + self, new_db_url: str, http_client: Optional[Any] = None + ) -> None: + """Recreate both writer and reader Prisma clients. + + The writer reconnect path in PrismaClient calls + `self.db.recreate_prisma_client(...)`. Without this method, a DB-wide + connectivity event would only re-create the writer; the reader engine + would stay broken and every routed read would fail. We always recreate + the writer first (its URL is the one passed in), then best-effort + recreate the reader. A reader failure flips `_reader_unavailable=True` + so reads transparently fall through to the writer. + """ + await self._writer.recreate_prisma_client(new_db_url, http_client=http_client) + try: + await self._recreate_reader(http_client=http_client) + self._reader_unavailable = False + except Exception as e: + self._reader_unavailable = True + verbose_proxy_logger.warning( + "Failed to recreate reader Prisma client: %s. " + "Reads will fall back to the writer until the reader recovers.", + e, + ) + + async def _recreate_reader(self, http_client: Optional[Any] = None) -> None: + """Resolve the reader URL and recreate its Prisma client. + + IAM-enabled readers regenerate their token (host/port/user came from + the parsed reader URL at construction time). Non-IAM readers reuse + the URL stored in `DATABASE_URL_READ_REPLICA`. + """ + if self._reader.iam_token_db_auth: + new_reader_url = self._reader.get_rds_iam_token() + if not new_reader_url: + raise RuntimeError( + "Failed to generate fresh IAM token for read replica" + ) + await self._reader.recreate_prisma_client( + new_reader_url, http_client=http_client + ) + return + reader_url = os.getenv("DATABASE_URL_READ_REPLICA", "") + if not reader_url: + raise RuntimeError( + "DATABASE_URL_READ_REPLICA not set; cannot recreate read replica client" + ) + await self._reader.recreate_prisma_client(reader_url, http_client=http_client) + + def __getattr__(self, name: str) -> Any: + if name in _TOP_LEVEL_READ_METHODS: + target = self._writer if self._reader_unavailable else self._reader + return getattr(target, name) + writer_attr = getattr(self._writer, name) + # Per-model action accessors are non-callable instances that expose + # both `find_many` and `create`. Methods like execute_raw / batch_ / + # tx are callables and stay on the writer untouched. + if ( + not callable(writer_attr) + and hasattr(writer_attr, "find_many") + and hasattr(writer_attr, "create") + ): + try: + reader_attr = getattr(self._reader, name) + except AttributeError: + return writer_attr + return _RoutedActions(writer_attr, reader_attr, self._should_use_reader) + return writer_attr diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 06fc0819a76..6359b48654b 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -813,7 +813,12 @@ def run_server( # noqa: PLR0915 from litellm.proxy.auth.rds_iam_token import generate_iam_auth_token db_host = os.getenv("DATABASE_HOST") - db_port = os.getenv("DATABASE_PORT") + # Default to the Postgres standard port. Without a default, + # `db_port=None` flows into `boto.generate_db_auth_token(Port=None)` + # and botocore stringifies it to `"None"` while building the + # presigned URL, which then blows up with `ValueError: Port could + # not be cast to integer value as 'None'` during signing. + db_port = os.getenv("DATABASE_PORT", "5432") db_user = os.getenv("DATABASE_USER") db_name = os.getenv("DATABASE_NAME") db_schema = os.getenv("DATABASE_SCHEMA") diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index a52dc8e55fb..0577110a26f 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -113,7 +113,11 @@ from litellm.proxy.db.exception_handler import ( call_with_db_reconnect_retry, ) from litellm.proxy.db.log_db_metrics import log_db_metrics -from litellm.proxy.db.prisma_client import PrismaWrapper +from litellm.proxy.db.prisma_client import ( + PrismaWrapper, + parse_iam_endpoint_from_url, +) +from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( UnifiedLLMGuardrails, ) @@ -2569,24 +2573,101 @@ class PrismaClient: raise Exception( "Unable to find Prisma binaries. Please run 'prisma generate' first." ) + iam_flag = ( + self.iam_token_db_auth if self.iam_token_db_auth is not None else False + ) + # When read-replica routing is on, tag log lines with [writer]/[reader] + # so the two wrappers' interleaved IAM refresh logs can be told apart. + # Single-DB deployments get an empty prefix (logs unchanged). + read_replica_url = os.getenv("DATABASE_URL_READ_REPLICA") + writer_log_prefix = "[writer]" if read_replica_url else "" if http_client is not None: - self.db = PrismaWrapper( + writer_wrapper = PrismaWrapper( original_prisma=Prisma(http=http_client), - iam_token_db_auth=( - self.iam_token_db_auth - if self.iam_token_db_auth is not None - else False - ), + iam_token_db_auth=iam_flag, + log_prefix=writer_log_prefix, ) else: - self.db = PrismaWrapper( + writer_wrapper = PrismaWrapper( original_prisma=Prisma(), - iam_token_db_auth=( - self.iam_token_db_auth - if self.iam_token_db_auth is not None - else False - ), - ) # Client to connect to Prisma db + iam_token_db_auth=iam_flag, + log_prefix=writer_log_prefix, + ) + + # Optional read-replica routing. When DATABASE_URL_READ_REPLICA is set, + # reads (find_*, count, group_by, query_raw/_first) are routed to the + # reader endpoint and writes stay on the writer. Falls back to the + # writer-only wrapper when the env var is unset, preserving existing + # single-DB deployments. + self.db: Union[PrismaWrapper, RoutingPrismaWrapper] + if read_replica_url: + try: + # If IAM auth is enabled, the reader refreshes its own token on + # the same cadence as the writer. We parse the static endpoint + # pieces (host/port/user/db) once from the reader URL — only + # the IAM token rotates after that. + reader_iam_endpoint = ( + parse_iam_endpoint_from_url(read_replica_url) if iam_flag else None + ) + # Mint a fresh IAM token for the reader BEFORE constructing the + # Prisma client. Mirrors what `proxy_cli.py` already does for + # the writer (proxy_cli.py:812-832) — without this, the reader + # Prisma is built with whatever placeholder URL the user + # supplied (no real token), and the first query falls through + # to the synchronous fallback path in + # `PrismaWrapper.__getattr__`, which deadlocks the event loop + # and times out after 30s. + if iam_flag and reader_iam_endpoint is not None: + from litellm.proxy.auth.rds_iam_token import ( + generate_iam_auth_token, + ) + + reader_token = generate_iam_auth_token( + db_host=reader_iam_endpoint.host, + db_port=reader_iam_endpoint.port, + db_user=reader_iam_endpoint.user, + ) + read_replica_url = reader_iam_endpoint.build_url(reader_token) + os.environ["DATABASE_URL_READ_REPLICA"] = read_replica_url + reader_kwargs: Dict[str, Any] = { + "datasource": {"url": read_replica_url} + } + if http_client is not None: + reader_prisma = Prisma(http=http_client, **reader_kwargs) + else: + reader_prisma = Prisma(**reader_kwargs) + reader_wrapper = PrismaWrapper( + original_prisma=reader_prisma, + iam_token_db_auth=iam_flag, + db_url_env_var="DATABASE_URL_READ_REPLICA", + iam_endpoint=reader_iam_endpoint, + recreate_uses_datasource=True, + log_prefix="[reader]", + ) + self.db = RoutingPrismaWrapper( + writer=writer_wrapper, reader=reader_wrapper + ) + verbose_proxy_logger.info( + "PrismaClient: read-replica routing enabled via DATABASE_URL_READ_REPLICA" + + (" (with IAM token auto-refresh)" if iam_flag else "") + ) + except Exception as e: + # Reader is opt-in; never let its construction fail proxy + # startup. Mirrors the runtime contract from + # `RoutingPrismaWrapper.connect`: reader-side failures are + # logged and we keep serving traffic via the writer alone. + # This recovers from transient AWS STS hiccups during the + # reader IAM token mint, malformed DATABASE_URL_READ_REPLICA, + # and Prisma construction errors. Operator restart is required + # to retry read-routing once the underlying issue is resolved. + verbose_proxy_logger.warning( + "Failed to initialize read replica Prisma client: %s. " + "Falling back to writer-only mode (no read routing) until proxy restart.", + e, + ) + self.db = writer_wrapper + else: + self.db = writer_wrapper # Client to connect to Prisma db self._db_reconnect_lock = asyncio.Lock() self._db_health_watchdog_task: Optional[asyncio.Task] = None self._db_last_reconnect_attempt_ts: float = 0.0 @@ -2624,6 +2705,13 @@ class PrismaClient: self._engine_wait_thread: Optional[threading.Thread] = None verbose_proxy_logger.debug("Success - Created Prisma Client") + @property + def writer_db(self) -> PrismaWrapper: + """Underlying writer Prisma wrapper, regardless of read-replica routing.""" + if isinstance(self.db, RoutingPrismaWrapper): + return self.db.writer + return self.db + def get_request_status( self, payload: Union[dict, SpendLogsPayload] ) -> Literal["success", "failure"]: @@ -4272,7 +4360,10 @@ class PrismaClient: self._cleanup_engine_watcher() await self.db.recreate_prisma_client(db_url) await self._start_engine_watcher() - await self.db.query_raw("SELECT 1") + # Smoke-test the writer specifically; query_raw on the routing + # wrapper sends to the reader, which would not validate the + # newly-recreated writer engine. + await self.writer_db.query_raw("SELECT 1") await asyncio.wait_for(_do_direct_reconnect(), timeout=effective_timeout) diff --git a/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py b/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py new file mode 100644 index 00000000000..8c3a2b9e2d7 --- /dev/null +++ b/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py @@ -0,0 +1,887 @@ +import asyncio +import logging +import os +import sys +from typing import Any, Dict +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../../..")) + + +# NOTE: do NOT patch sys.modules["prisma"] file-wide via an autouse fixture. +# Doing so leaks across pytest-xdist test scheduling: when a worker runs a +# routing test, then later runs test_exception_handler.py, the cached MagicMock +# attribute references break `isinstance(e, prisma.errors.X)` in +# `is_database_transport_error`. The two tests below that actually need to +# stub the prisma SDK do so per-test via monkeypatch, which is properly scoped. + + +def _make_wrappers(): + from litellm.proxy.db.prisma_client import PrismaWrapper + + writer_inner = MagicMock(name="writer_prisma") + reader_inner = MagicMock(name="reader_prisma") + writer = PrismaWrapper(original_prisma=writer_inner, iam_token_db_auth=False) + reader = PrismaWrapper(original_prisma=reader_inner, iam_token_db_auth=False) + return writer, writer_inner, reader, reader_inner + + +class _FakeActions: + """Stand-in for a Prisma per-model Actions instance (non-callable, has find_many/create).""" + + def __init__(self, name: str): + self._name = name + for method in ( + "find_many", + "find_unique", + "find_first", + "count", + "group_by", + "create", + "update", + "upsert", + "delete", + "delete_many", + "update_many", + ): + setattr(self, method, MagicMock(name=f"{name}.{method}")) + + +def _model_actions_mock(name: str) -> _FakeActions: + return _FakeActions(name) + + +def test_top_level_query_raw_routes_to_reader(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + # query_raw should resolve to the reader's underlying client. + assert routing.query_raw is reader_inner.query_raw + assert routing.query_first is reader_inner.query_first + + +def test_top_level_execute_raw_routes_to_writer(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + # execute_raw, batch_, tx are write-side and must hit the writer. + assert routing.execute_raw is writer_inner.execute_raw + assert routing.batch_ is writer_inner.batch_ + assert routing.tx is writer_inner.tx + + +def test_per_model_reads_route_to_reader_writes_to_writer(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.litellm_usertable = _model_actions_mock("writer_users") + reader_inner.litellm_usertable = _model_actions_mock("reader_users") + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + actions = routing.litellm_usertable + + # Reads → reader actions. + assert actions.find_many is reader_inner.litellm_usertable.find_many + assert actions.find_unique is reader_inner.litellm_usertable.find_unique + assert actions.find_first is reader_inner.litellm_usertable.find_first + assert actions.count is reader_inner.litellm_usertable.count + assert actions.group_by is reader_inner.litellm_usertable.group_by + + # Writes → writer actions. + assert actions.create is writer_inner.litellm_usertable.create + assert actions.update is writer_inner.litellm_usertable.update + assert actions.upsert is writer_inner.litellm_usertable.upsert + assert actions.delete is writer_inner.litellm_usertable.delete + assert actions.update_many is writer_inner.litellm_usertable.update_many + assert actions.delete_many is writer_inner.litellm_usertable.delete_many + + +@pytest.mark.asyncio +async def test_connect_invokes_both_clients(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.connect = AsyncMock() + reader_inner.connect = AsyncMock() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + await routing.connect() + + writer_inner.connect.assert_awaited_once() + reader_inner.connect.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_connect_logs_writer_and_reader_success(caplog): + """Successful startup emits a positive INFO confirmation for both writer + and reader so operators can verify connectivity without inspecting the URL + in logs.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.connect = AsyncMock() + reader_inner.connect = AsyncMock() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"): + await routing.connect() + + messages = [r.getMessage() for r in caplog.records] + assert "[writer] DB connected" in messages + assert "[reader] DB connected" in messages + + +@pytest.mark.asyncio +async def test_disconnect_continues_when_one_side_fails(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.disconnect = AsyncMock(side_effect=RuntimeError("writer down")) + reader_inner.disconnect = AsyncMock() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + with pytest.raises(RuntimeError, match="writer down"): + await routing.disconnect() + + # Reader still attempted even though writer raised. + reader_inner.disconnect.assert_awaited_once() + + +def test_is_connected_reflects_writer_only(): + """is_connected() must NOT depend on reader health — a healthy writer with + a degraded reader should report True so that PrismaClient.connect()'s + health check does not re-trigger a writer reconnect (which only fixes + writer-side problems and would loop indefinitely).""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + writer_inner.is_connected = MagicMock(return_value=True) + reader_inner.is_connected = MagicMock(return_value=True) + assert routing.is_connected() is True + + # Reader down → still True (reader degradation is tracked separately). + reader_inner.is_connected = MagicMock(return_value=False) + assert routing.is_connected() is True + + # Writer down → False. + writer_inner.is_connected = MagicMock(return_value=False) + assert routing.is_connected() is False + + +def test_token_refresh_delegates_to_both_writer_and_reader(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.start_token_refresh_task = AsyncMock() + writer.stop_token_refresh_task = AsyncMock() + reader = MagicMock() + reader.start_token_refresh_task = AsyncMock() + reader.stop_token_refresh_task = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + asyncio.run(routing.start_token_refresh_task()) + asyncio.run(routing.stop_token_refresh_task()) + + # Both wrappers get start/stop — each manages its own IAM token. When + # IAM is disabled on a wrapper its task body is a no-op. + writer.start_token_refresh_task.assert_awaited_once() + writer.stop_token_refresh_task.assert_awaited_once() + reader.start_token_refresh_task.assert_awaited_once() + reader.stop_token_refresh_task.assert_awaited_once() + + +def test_routed_actions_falls_back_to_writer_for_unknown_methods(): + from litellm.proxy.db.routing_prisma_wrapper import _RoutedActions + + writer_actions = _model_actions_mock("writer") + writer_actions.some_custom_method = "writer-custom" + reader_actions = _model_actions_mock("reader") + reader_actions.some_custom_method = "reader-custom" + + routed = _RoutedActions(writer_actions, reader_actions, lambda: True) + # Unknown method → defaults to writer (safe fallback for write-like ops). + assert routed.some_custom_method == "writer-custom" + + +def test_routed_actions_respects_should_use_reader_flag(): + """When the routing wrapper marks the reader unavailable, _RoutedActions + must redirect reads to the writer instead — without needing to re-fetch + the actions accessor.""" + from litellm.proxy.db.routing_prisma_wrapper import _RoutedActions + + writer_actions = _model_actions_mock("writer") + reader_actions = _model_actions_mock("reader") + + use_reader = {"value": True} + routed = _RoutedActions(writer_actions, reader_actions, lambda: use_reader["value"]) + + # Reader healthy → reads to reader. + assert routed.find_many is reader_actions.find_many + + # Reader degrades mid-flight → next read goes to writer. + use_reader["value"] = False + assert routed.find_many is writer_actions.find_many + + +# --------------------------------------------------------------------------- +# Reader graceful degradation +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_connect_swallows_reader_failure_and_falls_back_to_writer(): + """A reader connect failure must NOT abort proxy startup. The wrapper + flips into degraded mode so subsequent reads route to the writer.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.connect = AsyncMock() + reader_inner.connect = AsyncMock(side_effect=RuntimeError("reader unreachable")) + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + # Must not raise — reader failure is non-fatal. + await routing.connect() + + assert routing.reader_unavailable is True + writer_inner.connect.assert_awaited_once() + reader_inner.connect.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_reads_route_to_writer_when_reader_unavailable(): + """Top-level read methods and per-model reads must fall through to the + writer while the reader is degraded.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.litellm_usertable = _model_actions_mock("writer_users") + reader_inner.litellm_usertable = _model_actions_mock("reader_users") + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + routing._reader_unavailable = True + + # Top-level reads → writer. + assert routing.query_raw is writer_inner.query_raw + assert routing.query_first is writer_inner.query_first + + # Per-model reads → writer actions. + actions = routing.litellm_usertable + assert actions.find_many is writer_inner.litellm_usertable.find_many + assert actions.find_unique is writer_inner.litellm_usertable.find_unique + + +@pytest.mark.asyncio +async def test_recreate_prisma_client_recreates_both_writer_and_reader(): + """Writer reconnect path calls recreate_prisma_client. The routing wrapper + must recreate BOTH clients so a DB-wide event doesn't leave a stale reader.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = False + reader.recreate_prisma_client = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + with patch.dict(os.environ, {"DATABASE_URL_READ_REPLICA": "reader-url"}): + await routing.recreate_prisma_client("writer-url", http_client=None) + + writer.recreate_prisma_client.assert_awaited_once_with( + "writer-url", http_client=None + ) + reader.recreate_prisma_client.assert_awaited_once_with( + "reader-url", http_client=None + ) + assert routing.reader_unavailable is False + + +@pytest.mark.asyncio +async def test_recreate_recovers_reader_after_prior_degradation(): + """If a previous connect/recreate degraded the reader, a successful + recreate must clear the flag so reads start hitting the reader again.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = False + reader.recreate_prisma_client = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + routing._reader_unavailable = True + + with patch.dict(os.environ, {"DATABASE_URL_READ_REPLICA": "reader-url"}): + await routing.recreate_prisma_client("writer-url") + + assert routing.reader_unavailable is False + + +@pytest.mark.asyncio +async def test_recreate_degrades_reader_if_reader_recreate_fails(): + """If the reader recreate fails, writer recreate still succeeds and the + routing wrapper degrades (does not raise).""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = False + reader.recreate_prisma_client = AsyncMock( + side_effect=RuntimeError("reader still down") + ) + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + with patch.dict(os.environ, {"DATABASE_URL_READ_REPLICA": "reader-url"}): + # Must not raise — writer was recreated, reader is best-effort. + await routing.recreate_prisma_client("writer-url") + + writer.recreate_prisma_client.assert_awaited_once() + assert routing.reader_unavailable is True + + +@pytest.mark.asyncio +async def test_recreate_degrades_reader_when_replica_url_missing(): + """Non-IAM reader needs DATABASE_URL_READ_REPLICA. If it's missing + (configuration drift), the wrapper degrades instead of raising.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = False + reader.recreate_prisma_client = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + # Ensure env var is absent. + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("DATABASE_URL_READ_REPLICA", None) + await routing.recreate_prisma_client("writer-url") + + writer.recreate_prisma_client.assert_awaited_once() + reader.recreate_prisma_client.assert_not_awaited() + assert routing.reader_unavailable is True + + +@pytest.mark.asyncio +async def test_recreate_iam_reader_refreshes_token(): + """IAM-enabled readers must refresh their token (reader has its own parsed + endpoint) and pass the fresh URL to recreate_prisma_client.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = True + reader.get_rds_iam_token = MagicMock(return_value="postgresql://u:fresh@h:5432/db") + reader.recreate_prisma_client = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + await routing.recreate_prisma_client("writer-url") + + reader.get_rds_iam_token.assert_called_once() + reader.recreate_prisma_client.assert_awaited_once_with( + "postgresql://u:fresh@h:5432/db", http_client=None + ) + assert routing.reader_unavailable is False + + +@pytest.mark.asyncio +async def test_recreate_degrades_when_iam_token_generation_returns_none(): + """If `get_rds_iam_token` returns None (e.g. AWS-side failure), the wrapper + must degrade rather than crash — this exercises the explicit `raise + RuntimeError` inside `_recreate_reader`'s IAM branch.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = True + reader.get_rds_iam_token = MagicMock(return_value=None) + reader.recreate_prisma_client = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + await routing.recreate_prisma_client("writer-url") + + writer.recreate_prisma_client.assert_awaited_once() + reader.recreate_prisma_client.assert_not_awaited() + assert routing.reader_unavailable is True + + +def test_writer_and_reader_properties_expose_underlying_wrappers(): + """The `writer` and `reader` properties are used by PrismaClient.writer_db + to smoke-test the writer specifically during reconnect — they must return + the exact wrappers passed in.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, _, reader, _ = _make_wrappers() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + assert routing.writer is writer + assert routing.reader is reader + + +def test_per_model_accessor_falls_back_when_reader_lacks_attr(): + """If the reader Prisma client somehow lacks a model accessor that the + writer has (older client / partial mock), the wrapper must fall back to + the writer accessor instead of raising AttributeError to the caller.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + # Plain class with only the accessor set on the writer side. Using a real + # class instead of MagicMock so attribute access raises AttributeError + # naturally instead of auto-creating mock attributes. + class _PartialPrisma: + pass + + writer_inner = _PartialPrisma() + writer_inner.litellm_usertable = _model_actions_mock("writer_users") + reader_inner = _PartialPrisma() # deliberately missing litellm_usertable + + writer = PrismaWrapper(original_prisma=writer_inner, iam_token_db_auth=False) + reader = PrismaWrapper(original_prisma=reader_inner, iam_token_db_auth=False) + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + actions = routing.litellm_usertable + # Falls back to the writer's accessor verbatim — not a _RoutedActions wrapper. + assert actions is writer_inner.litellm_usertable + + +@pytest.mark.asyncio +async def test_writer_recreate_passes_http_client_through(monkeypatch): + """When PrismaClient is constructed with an http_client, recreate must + forward it to the new Prisma() so connection settings persist across + reconnects.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + captured_kwargs: Dict[str, Any] = {} + + class FakePrisma: + def __init__(self, **kwargs): + captured_kwargs.update(kwargs) + + async def connect(self): + return None + + fake_module = MagicMock() + fake_module.Prisma = FakePrisma + monkeypatch.setitem(sys.modules, "prisma", fake_module) + + writer = PrismaWrapper(original_prisma=MagicMock(), iam_token_db_auth=False) + sentinel_http = object() + await writer.recreate_prisma_client( + "postgresql://u:p@h:5432/db", http_client=sentinel_http + ) + + assert captured_kwargs == {"http": sentinel_http} + + +# --------------------------------------------------------------------------- +# IAM endpoint parsing + reader IAM refresh +# --------------------------------------------------------------------------- + + +def test_parse_iam_endpoint_from_url_extracts_all_fields(): + from litellm.proxy.db.prisma_client import parse_iam_endpoint_from_url + + ep = parse_iam_endpoint_from_url( + "postgresql://litellm_user:initial-token@aurora-reader.example.com:6543/litellm?schema=public" + ) + assert ep.host == "aurora-reader.example.com" + assert ep.port == "6543" + assert ep.user == "litellm_user" + assert ep.name == "litellm" + assert ep.schema == "public" + + +def test_parse_iam_endpoint_defaults_port_to_5432_and_skips_schema(): + from litellm.proxy.db.prisma_client import parse_iam_endpoint_from_url + + ep = parse_iam_endpoint_from_url("postgresql://u@host/dbname") + assert ep.host == "host" + assert ep.port == "5432" + assert ep.user == "u" + assert ep.name == "dbname" + assert ep.schema is None + + +def test_parse_iam_endpoint_rejects_url_without_user_or_dbname(): + from litellm.proxy.db.prisma_client import parse_iam_endpoint_from_url + + with pytest.raises(ValueError, match="missing host or username"): + parse_iam_endpoint_from_url("postgresql://host:5432/db") + with pytest.raises(ValueError, match="missing database name"): + parse_iam_endpoint_from_url("postgresql://u@host:5432/") + + +def test_iam_endpoint_build_url_inserts_token_verbatim(): + from litellm.proxy.db.prisma_client import IAMEndpoint + + # `generate_iam_auth_token` already URL-encodes the presigned token, so + # `build_url` must NOT encode again — double-encoding turned `%3D` into + # `%253D` and broke RDS auth on the reader path. + ep = IAMEndpoint(host="h", port="5432", user="u", name="db", schema="public") + pre_encoded_token = "token%2Fwith%3Fweird%26chars%3Dyes" + url = ep.build_url(pre_encoded_token) + assert url == f"postgresql://u:{pre_encoded_token}@h:5432/db?schema=public" + # Sanity check: no `%25` (the encoding of `%`), confirming we didn't re-encode. + assert "%25" not in url + + +@pytest.mark.asyncio +async def test_iam_refresh_logs_carry_log_prefix(caplog): + """When `log_prefix` is set on a PrismaWrapper, every IAM-related log + line emitted by that wrapper must start with the prefix so writer and + reader can be told apart in interleaved output.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + wrapper = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + log_prefix="[reader]", + ) + + with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"): + await wrapper.start_token_refresh_task() + # Loop emits "RDS IAM token refresh loop started..." on first tick. + # Cancel immediately so the loop body runs once and we can assert. + await wrapper.stop_token_refresh_task() + + messages = [r.getMessage() for r in caplog.records] + # Both start and stop notifications carry the prefix. + assert any( + m.startswith("[reader] Started RDS IAM token proactive refresh") + for m in messages + ) + assert any( + m.startswith("[reader] Stopped RDS IAM token refresh background task") + for m in messages + ) + + +def test_get_rds_iam_token_returns_none_when_iam_disabled(): + """`get_rds_iam_token` short-circuits to None when iam_token_db_auth is + False — covers the early-return guard at the top of the method.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + wrapper = PrismaWrapper(original_prisma=MagicMock(), iam_token_db_auth=False) + assert wrapper.get_rds_iam_token() is None + + +@pytest.mark.asyncio +async def test_getattr_does_not_block_inside_running_loop_on_expired_token(monkeypatch): + """When `__getattr__` runs inside a running event loop and the IAM token + is expired, it MUST schedule the refresh as a background task and return + immediately. The previous `run_coroutine_threadsafe` + `future.result()` + pattern deadlocks the loop (loop thread blocks waiting for a coroutine + that needs the loop to run) and times out at 30s — exactly what was + breaking the reader on first query.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + # Stale URL — `is_token_expired` returns True because the password isn't + # a parseable IAM token, so we exercise the expired branch. + monkeypatch.setenv( + "DATABASE_URL_READ_REPLICA", + "postgresql://reader:placeholder@reader.aurora.local:5432/litellm", + ) + + inner = MagicMock() + inner.query_raw = MagicMock(name="query_raw_attr") + + wrapper = PrismaWrapper( + original_prisma=inner, + iam_token_db_auth=True, + db_url_env_var="DATABASE_URL_READ_REPLICA", + ) + + # Replace the heavy refresh coroutine with a no-op AsyncMock so we can + # observe whether it was scheduled without actually doing the recreate. + refresh_calls = {"count": 0} + + async def fake_refresh(): + refresh_calls["count"] += 1 + + monkeypatch.setattr(wrapper, "_safe_refresh_token", fake_refresh) + + # Direct attribute access from inside this async test runs __getattr__ + # on the loop thread, exercising the in-loop branch. If the previous + # `run_coroutine_threadsafe` + `future.result()` pattern were back, this + # line would deadlock the loop and the test would hang (and pytest's + # per-test timeout would catch it). + attr = wrapper.query_raw + # Yield once so the scheduled refresh task gets a chance to run. + await asyncio.sleep(0) + + assert attr is inner.query_raw + assert refresh_calls["count"] == 1 + + +def test_writer_get_rds_iam_token_defaults_port_when_unset(monkeypatch): + """When DATABASE_PORT is unset, the writer must default to the Postgres + standard port instead of passing `None` through. Passing None to + `generate_iam_auth_token` makes botocore embed the literal string + \"None\" in the presigned URL during signing and crashes with + `ValueError: Port could not be cast to integer value as 'None'`.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + monkeypatch.setenv("DATABASE_HOST", "writer.aurora.local") + monkeypatch.delenv("DATABASE_PORT", raising=False) + monkeypatch.setenv("DATABASE_USER", "litellm") + monkeypatch.setenv("DATABASE_NAME", "litellm") + monkeypatch.delenv("DATABASE_SCHEMA", raising=False) + monkeypatch.delenv("DATABASE_URL", raising=False) + + captured: Dict[str, Any] = {} + + def fake_generate(db_host=None, db_port=None, db_user=None): + captured["port"] = db_port + return "TOKEN" + + fake_module = MagicMock() + fake_module.generate_iam_auth_token = fake_generate + monkeypatch.setitem(sys.modules, "litellm.proxy.auth.rds_iam_token", fake_module) + + writer = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + ) + new_url = writer.get_rds_iam_token() + + assert captured["port"] == "5432" # default applied, NOT None + assert ":5432/litellm" in (new_url or "") + + +def test_writer_get_rds_iam_token_uses_database_host_env_vars(monkeypatch): + """Writer's IAM path (no iam_endpoint configured) reads host/port/user/db + from the legacy DATABASE_HOST/PORT/USER/NAME env vars and writes the URL + back to DATABASE_URL — this is the pre-read-replica behavior the patch + must preserve.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + monkeypatch.setenv("DATABASE_HOST", "writer.aurora.local") + monkeypatch.setenv("DATABASE_PORT", "5432") + monkeypatch.setenv("DATABASE_USER", "litellm") + monkeypatch.setenv("DATABASE_NAME", "litellm") + monkeypatch.setenv("DATABASE_SCHEMA", "public") + monkeypatch.delenv("DATABASE_URL", raising=False) + + captured: Dict[str, Any] = {} + + def fake_generate(db_host=None, db_port=None, db_user=None): + captured["host"] = db_host + captured["port"] = db_port + captured["user"] = db_user + return "WRITER-TOKEN" + + fake_module = MagicMock() + fake_module.generate_iam_auth_token = fake_generate + monkeypatch.setitem(sys.modules, "litellm.proxy.auth.rds_iam_token", fake_module) + + writer = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + # No iam_endpoint → legacy DATABASE_HOST/etc. path. + ) + new_url = writer.get_rds_iam_token() + + assert captured == { + "host": "writer.aurora.local", + "port": "5432", + "user": "litellm", + } + assert new_url == ( + "postgresql://litellm:WRITER-TOKEN@writer.aurora.local:5432/litellm?schema=public" + ) + # Writer updates its own env var (DATABASE_URL by default), not the reader's. + assert os.environ["DATABASE_URL"] == new_url + + +def test_reader_iam_refresh_uses_parsed_endpoint(monkeypatch): + """The reader generates fresh tokens against its parsed endpoint and + writes the new URL to DATABASE_URL_READ_REPLICA — not DATABASE_URL.""" + from litellm.proxy.db.prisma_client import IAMEndpoint, PrismaWrapper + + # Pre-seed env vars so we can prove the reader does NOT touch DATABASE_URL. + monkeypatch.setenv("DATABASE_URL", "writer-url-untouched") + monkeypatch.setenv("DATABASE_URL_READ_REPLICA", "stale-reader-url") + + captured: Dict[str, Any] = {} + + def fake_generate(db_host=None, db_port=None, db_user=None): + captured["host"] = db_host + captured["port"] = db_port + captured["user"] = db_user + return "FRESH-TOKEN" + + fake_module = MagicMock() + fake_module.generate_iam_auth_token = fake_generate + monkeypatch.setitem(sys.modules, "litellm.proxy.auth.rds_iam_token", fake_module) + + endpoint = IAMEndpoint( + host="reader.aurora.local", + port="5432", + user="lit", + name="litellm", + schema=None, + ) + reader = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + db_url_env_var="DATABASE_URL_READ_REPLICA", + iam_endpoint=endpoint, + recreate_uses_datasource=True, + ) + + new_url = reader.get_rds_iam_token() + + # IAM token generator was called with the reader's parsed endpoint, not + # the writer's DATABASE_HOST/PORT/USER env vars. + assert captured == { + "host": "reader.aurora.local", + "port": "5432", + "user": "lit", + } + assert new_url is not None + assert new_url.startswith( + "postgresql://lit:FRESH-TOKEN@reader.aurora.local:5432/litellm" + ) + # The reader updates its OWN env var; writer's DATABASE_URL is left alone. + assert os.environ["DATABASE_URL_READ_REPLICA"] == new_url + assert os.environ["DATABASE_URL"] == "writer-url-untouched" + + +@pytest.mark.asyncio +async def test_reader_recreate_uses_datasource_override(monkeypatch): + """Reader recreate must pass `datasource={"url": ...}` to Prisma() — Prisma + only auto-reads DATABASE_URL, so without the override the new reader URL + would be silently ignored.""" + from litellm.proxy.db.prisma_client import IAMEndpoint, PrismaWrapper + + captured_kwargs: Dict[str, Any] = {} + + class FakePrisma: + def __init__(self, **kwargs): + captured_kwargs.update(kwargs) + + async def connect(self): + return None + + fake_module = MagicMock() + fake_module.Prisma = FakePrisma + monkeypatch.setitem(sys.modules, "prisma", fake_module) + + reader = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + db_url_env_var="DATABASE_URL_READ_REPLICA", + iam_endpoint=IAMEndpoint(host="h", port="5432", user="u", name="db"), + recreate_uses_datasource=True, + ) + + await reader.recreate_prisma_client( + "postgresql://u:newtoken@h:5432/db", http_client=None + ) + + assert captured_kwargs == { + "datasource": {"url": "postgresql://u:newtoken@h:5432/db"} + } + + +@pytest.mark.asyncio +async def test_writer_recreate_does_not_use_datasource(monkeypatch): + """Writer keeps relying on Prisma reading DATABASE_URL from env — datasource + override must NOT leak into the writer path (would override the freshly + rotated env var).""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + captured_kwargs: Dict[str, Any] = {} + + class FakePrisma: + def __init__(self, **kwargs): + captured_kwargs.update(kwargs) + + async def connect(self): + return None + + fake_module = MagicMock() + fake_module.Prisma = FakePrisma + monkeypatch.setitem(sys.modules, "prisma", fake_module) + + writer = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + ) + + await writer.recreate_prisma_client( + "postgresql://u:newtoken@h:5432/db", http_client=None + ) + + assert "datasource" not in captured_kwargs + + +def test_prisma_client_init_falls_back_to_writer_when_reader_iam_token_fails( + monkeypatch, caplog +): + """A transient AWS STS error (or any other failure) during the reader + IAM token mint must NOT abort proxy startup. The reader is opt-in, so + `PrismaClient.__init__` should log a warning and fall back to the + writer-only `PrismaWrapper`. The runtime contract in + `RoutingPrismaWrapper.connect` already says reader-side failures are + non-fatal — but that code never runs if construction throws first.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + monkeypatch.setenv( + "DATABASE_URL_READ_REPLICA", + "postgresql://reader_user@reader.aurora.local:5432/litellm", + ) + + class FakePrisma: + def __init__(self, **kwargs): + self.kwargs = kwargs + + async def connect(self): + return None + + fake_prisma_module = MagicMock() + fake_prisma_module.Prisma = FakePrisma + monkeypatch.setitem(sys.modules, "prisma", fake_prisma_module) + + fake_iam_module = MagicMock() + + def boom(**_kwargs): + raise RuntimeError("simulated AWS STS hiccup") + + fake_iam_module.generate_iam_auth_token = boom + monkeypatch.setitem( + sys.modules, "litellm.proxy.auth.rds_iam_token", fake_iam_module + ) + + from litellm.proxy.utils import PrismaClient + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + client = PrismaClient( + database_url="postgresql://writer@writer.aurora.local:5432/litellm", + proxy_logging_obj=MagicMock(), + ) + + # Construction did not raise, and the proxy is in writer-only mode — + # NOT a RoutingPrismaWrapper, so reads will go to the writer. + assert isinstance(client.db, PrismaWrapper) + assert not isinstance(client.db, RoutingPrismaWrapper) + # And the operator gets a clear warning. + assert any( + "Failed to initialize read replica Prisma client" in r.getMessage() + for r in caplog.records + ) From 0d551ac4f06b62d270694ee877fdeccf329f5528 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 8 May 2026 21:08:55 -0700 Subject: [PATCH 36/85] fix(proxy): reserve per-image cost for image-generation requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Image-generation routes (dall-e-3, flux, etc.) have no per-token output cost so they fell through to the no-reservation read-time-only path. Concurrent image requests against a depleted budget could all pass common_checks (counter exactly at max_budget passes the strict-`>` gate) and reach the provider before reconciliation caught up. Add per-image reservation in _estimate_request_max_cost_for_model: when the model has a per-image cost field, reserve `n × cost_per_image` upfront. The atomic counter increment serializes concurrent admissions, so the second request sees the post-first-reservation counter and raises BudgetExceededError instead of silently leaking through. Both `output_cost_per_image` and `input_cost_per_image` are honored — naming is inconsistent across providers (OpenAI dall-e-3 uses input_cost_per_image, aiml/dall-e-3 uses output_cost_per_image for the same per-generated-image price). Per-pixel pricing (DALL-E 2 size variants) and TTS/STT routes still fall through to read-time enforcement; those are follow-ups. --- .../spend_tracking/budget_reservation.py | 41 +++++ .../proxy/test_budget_reservation.py | 148 ++++++++++++++++++ 2 files changed, 189 insertions(+) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index feae368d23f..1b11af84d0d 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -827,6 +827,13 @@ def _estimate_request_max_cost_for_model( if model_info is None: return None + image_cost = _estimate_image_generation_cost( + request_body=request_body, + model_info=model_info, + ) + if image_cost is not None: + return image_cost + input_cost_per_token = _to_float(model_info.get("input_cost_per_token")) output_cost_per_token = _to_float(model_info.get("output_cost_per_token")) input_tokens = _estimate_input_tokens( @@ -858,6 +865,40 @@ def _estimate_request_max_cost_for_model( return cost +def _estimate_image_generation_cost( + request_body: dict, + model_info: Dict[str, Any], +) -> Optional[float]: + """ + Reserve `n × per-image cost` for image-generation requests so concurrent + requests against a depleted budget cannot all slip past the admission gate + onto the provider. Token-based pricing (e.g. gpt-image-1) is handled by + the chat-route token path; per-pixel and size/quality-tiered pricing + (DALL-E 2 size variants, premium tiers) are not handled here and fall + through to read-time enforcement. + + The "output" vs "input" cost-per-image naming is inconsistent across + providers — OpenAI's dall-e-3 entry uses ``input_cost_per_image`` while + aiml/dall-e-3 uses ``output_cost_per_image`` — so both are summed. + """ + output_cost_per_image = _to_float(model_info.get("output_cost_per_image")) + input_cost_per_image = _to_float(model_info.get("input_cost_per_image")) + is_image_gen = ( + model_info.get("mode") == "image_generation" + or output_cost_per_image is not None + or input_cost_per_image is not None + ) + if not is_image_gen: + return None + + cost_per_image = (output_cost_per_image or 0.0) + (input_cost_per_image or 0.0) + if cost_per_image <= 0: + return None + + n = _to_int(request_body.get("n")) or 1 + return cost_per_image * max(n, 1) + + def _get_model_cost_info( model: str, llm_router: Optional[Router], diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 0e2ca98a113..751e58e7421 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -699,6 +699,154 @@ async def test_should_clamp_reservation_to_model_ceiling_when_caller_overrequest await release_budget_reservation(reservation) +@pytest.mark.asyncio +async def test_should_reserve_image_generation_cost_per_image( + spend_counter_state, +): + """Image-generation requests reserve `n × per-image cost` so concurrent + requests against a depleted budget cannot all bypass the admission gate. + The OpenAI ``dall-e-3`` entry exposes the per-image price as + ``input_cost_per_image`` (a naming quirk), while other providers use + ``output_cost_per_image`` — both must be honored.""" + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + valid_token = UserAPIKeyAuth( + token="key-image-gen", + spend=0.0, + max_budget=10.0, + ) + await key_cache.async_set_cache(key="key-image-gen", value=valid_token) + + request_body = {"model": "dall-e-3", "prompt": "a cat", "n": 3} + + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "mode": "image_generation", + "input_cost_per_image": 0.04, + }, + ): + reservation = await reserve_budget_for_request( + request_body=request_body, + route="/v1/images/generations", + llm_router=None, + valid_token=valid_token, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + assert reservation is not None + assert reservation["reserved_cost"] == pytest.approx(0.12) # 3 × $0.04 + await release_budget_reservation(reservation) + + +@pytest.mark.asyncio +async def test_should_reject_concurrent_image_request_against_depleted_budget( + spend_counter_state, +): + """Greptile P1 regression: with image-gen reservation in place, a second + concurrent image request against a budget already pinned at the cap by + the first reservation must raise BudgetExceededError instead of + silently reaching the provider.""" + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + valid_token = UserAPIKeyAuth( + token="key-image-deplete", + spend=0.0, + team_id="team-image-deplete", + ) + team_object = LiteLLM_TeamTable( + team_id="team-image-deplete", + max_budget=0.04, + spend=0.0, + ) + await key_cache.async_set_cache( + key=f"team_id:{team_object.team_id}", + value=team_object, + ) + + request_body = {"model": "dall-e-3", "prompt": "a cat"} + + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "mode": "image_generation", + "input_cost_per_image": 0.04, + }, + ): + first = await reserve_budget_for_request( + request_body=request_body, + route="/v1/images/generations", + llm_router=None, + valid_token=valid_token, + team_object=team_object, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + assert first is not None + + with pytest.raises(litellm.BudgetExceededError): + await reserve_budget_for_request( + request_body=request_body, + route="/v1/images/generations", + llm_router=None, + valid_token=valid_token, + team_object=team_object, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + await release_budget_reservation(first) + + +@pytest.mark.asyncio +async def test_should_skip_reservation_for_per_pixel_image_model( + spend_counter_state, +): + """DALL-E 2-style per-pixel pricing depends on the requested ``size``, + which we don't decode here. Fall through to read-time enforcement + rather than guess.""" + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + valid_token = UserAPIKeyAuth( + token="key-image-per-pixel", + spend=0.0, + max_budget=1.0, + ) + await key_cache.async_set_cache(key="key-image-per-pixel", value=valid_token) + + request_body = {"model": "dall-e-2", "prompt": "a cat", "size": "256x256"} + + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "mode": "image_generation", + "input_cost_per_pixel": 2.4414e-07, + "output_cost_per_pixel": 0.0, + }, + ): + reservation = await reserve_budget_for_request( + request_body=request_body, + route="/v1/images/generations", + llm_router=None, + valid_token=valid_token, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + assert reservation is None + + def test_should_start_window_without_reset_at_at_duration_boundary(): before = datetime.now(timezone.utc) - timedelta(hours=1) From 963cb4694dd39f43d01e9d02da03784dbb0b5fb7 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 9 May 2026 09:16:27 -0700 Subject: [PATCH 37/85] fix(proxy): gate image-gen reservation strictly on model mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous detection treated any model with input_cost_per_image or output_cost_per_image as image generation. Several chat and embedding models carry those fields to price multimodal vision input, not generated images: - gemini-3.1-pro-preview (mode=chat) has output_cost_per_image=0.00012 alongside input/output token pricing. - azure/gpt-realtime-* (mode=chat) has input_cost_per_image=5e-6. - amazon.titan-embed-image-v1 (mode=embedding) has input_cost_per_image=6e-5. For these models the image-gen branch fired first and reserved a fraction of a cent per request, short-circuiting the token-priced path entirely. Long Gemini chats reserved 1 × $0.00012 instead of the true token cost. Gate strictly on mode in {"image_generation", "image_edit"}. All 197 real image_generation entries and all 31 image_edit entries (Flux Kontext, Stability inpaint/outpaint, etc.) carry the right mode, so the field-presence fallback was unnecessary. Adds regression tests for the chat-model-with-image-cost-field case and for image_edit reservation. --- .../spend_tracking/budget_reservation.py | 20 ++-- .../proxy/test_budget_reservation.py | 104 ++++++++++++++++++ 2 files changed, 116 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 1b11af84d0d..200a17e2368 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -881,16 +881,20 @@ def _estimate_image_generation_cost( providers — OpenAI's dall-e-3 entry uses ``input_cost_per_image`` while aiml/dall-e-3 uses ``output_cost_per_image`` — so both are summed. """ - output_cost_per_image = _to_float(model_info.get("output_cost_per_image")) - input_cost_per_image = _to_float(model_info.get("input_cost_per_image")) - is_image_gen = ( - model_info.get("mode") == "image_generation" - or output_cost_per_image is not None - or input_cost_per_image is not None - ) - if not is_image_gen: + # Gate strictly on `mode`. Several chat and embedding models carry + # ``input_cost_per_image`` / ``output_cost_per_image`` to price multimodal + # *vision input* (e.g. ``gemini-3.1-pro-preview``, ``azure/gpt-realtime-*``, + # ``amazon.titan-embed-image-v1``). Falling back to "treat as image-gen if + # an image cost field is present" would short-circuit the token-priced + # path for those models and reserve a fraction of a cent instead of the + # true per-token cost. All real image-generation entries in + # ``model_prices_and_context_window.json`` carry ``mode: image_generation`` + # or ``mode: image_edit``, so the field-presence fallback is unnecessary. + if model_info.get("mode") not in ("image_generation", "image_edit"): return None + output_cost_per_image = _to_float(model_info.get("output_cost_per_image")) + input_cost_per_image = _to_float(model_info.get("input_cost_per_image")) cost_per_image = (output_cost_per_image or 0.0) + (input_cost_per_image or 0.0) if cost_per_image <= 0: return None diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 751e58e7421..aa0f8d63274 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -847,6 +847,110 @@ async def test_should_skip_reservation_for_per_pixel_image_model( assert reservation is None +@pytest.mark.asyncio +async def test_should_use_token_pricing_for_chat_model_with_image_cost_field( + spend_counter_state, +): + """Several chat and embedding models carry ``input_cost_per_image`` / + ``output_cost_per_image`` to price multimodal vision *input*, not image + generation (e.g. gemini-3.1-pro-preview, azure/gpt-realtime-*, + amazon.titan-embed-image-v1). _estimate_image_generation_cost must gate + on ``mode`` so these models still go through the token-priced path — + otherwise a long chat reserves a fraction of a cent instead of the true + token cost.""" + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + valid_token = UserAPIKeyAuth( + token="key-multimodal-chat", + spend=0.0, + max_budget=10.0, + ) + await key_cache.async_set_cache(key="key-multimodal-chat", value=valid_token) + + # Roughly the gemini-3.1-pro-preview shape: chat-mode model that + # carries an output_cost_per_image alongside token pricing. + output_cost_per_token = 1.2e-5 + request_body = { + "model": "gemini-3.1-pro-preview", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 1000, + } + expected_cost = 1000 * output_cost_per_token # token-priced path, not 1 × $0.00012 + + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "mode": "chat", + "input_cost_per_token": 2e-6, + "output_cost_per_token": output_cost_per_token, + "output_cost_per_image": 0.00012, + "max_output_tokens": 64000, + }, + ): + reservation = await reserve_budget_for_request( + request_body=request_body, + route="/chat/completions", + llm_router=None, + valid_token=valid_token, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + assert reservation is not None + # Token-priced path: reservation ≈ output_tokens × output_cost_per_token, + # plus a small input-token contribution. Must NOT collapse to the + # per-image price ($0.00012) which would indicate the image-gen branch + # incorrectly fired for this chat model. + assert reservation["reserved_cost"] == pytest.approx(expected_cost, rel=0.05) + assert reservation["reserved_cost"] > 0.001 # well above per-image price + await release_budget_reservation(reservation) + + +@pytest.mark.asyncio +async def test_should_reserve_image_edit_cost_per_image( + spend_counter_state, +): + """``image_edit`` models (Flux Kontext, Stability inpaint/outpaint, etc.) + bill per generated image just like ``image_generation`` and must get + the same atomic per-image reservation.""" + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + valid_token = UserAPIKeyAuth( + token="key-image-edit", + spend=0.0, + max_budget=10.0, + ) + await key_cache.async_set_cache(key="key-image-edit", value=valid_token) + + request_body = {"model": "stability/inpaint", "prompt": "a cat", "n": 2} + + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "mode": "image_edit", + "output_cost_per_image": 0.05, + }, + ): + reservation = await reserve_budget_for_request( + request_body=request_body, + route="/v1/images/edits", + llm_router=None, + valid_token=valid_token, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + assert reservation is not None + assert reservation["reserved_cost"] == pytest.approx(0.10) # 2 × $0.05 + await release_budget_reservation(reservation) + + def test_should_start_window_without_reset_at_at_duration_boundary(): before = datetime.now(timezone.utc) - timedelta(hours=1) From 0f11b3e187b9662bdc51262c7f5cbd8a141f4936 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 9 May 2026 09:36:49 -0700 Subject: [PATCH 38/85] [UI] Rename "Default" key type to "Full Access" and reorder dropdown (#27218) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Create New Key dialog had a "Default" key-type option that was misleading: it was not the actual default selection (AI APIs is), and it grants access to all routes — broader than the description implied. Rename the option to "Full Access" with an accurate description, and reorder the dropdown to AI APIs → Management → Full Access. Switch the inner labels to antd Typography for consistency with the rest of the UI. UI-only change; the underlying enum value ("default") is unchanged so the API contract is preserved. --- .../organisms/create_key_button.tsx | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 29050907503..7d3f077dafc 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -9,7 +9,7 @@ import { formatNumberWithCommas } from "@/utils/dataUtils"; import { InfoCircleOutlined } from "@ant-design/icons"; import { useQueryClient } from "@tanstack/react-query"; import { Accordion, AccordionBody, AccordionHeader, Button, Col, Grid, Text, TextInput, Title } from "@tremor/react"; -import { Button as Button2, Form, Input, Modal, Radio, Select, Switch, Tag, Tooltip } from "antd"; +import { Button as Button2, Form, Input, Modal, Radio, Select, Switch, Tag, Tooltip, Typography } from "antd"; import debounce from "lodash/debounce"; import React, { useCallback, useEffect, useState } from "react"; import { rolesWithWriteAccess } from "../../utils/roles"; @@ -979,28 +979,28 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp } }} > - + From 7057d53862c557cf41a47d664bf039d1764d14e8 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 9 May 2026 10:13:00 -0700 Subject: [PATCH 39/85] fix(ui-tests): add Typography to antd mock in create_key_button test The antd mock omits Typography, which caused all 15 tests in create_key_button.test.tsx to fail with "No 'Typography' export is defined on the 'antd' mock" after #27218 switched the key-type dropdown labels to Typography.Text / Typography.Paragraph. Add Typography (with .Text, .Paragraph, .Title subcomponents) to the mock so the dropdown renders in the test environment. --- .../components/organisms/create_key_button.test.tsx | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.test.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.test.tsx index 3ad59cb3693..bd139df4833 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.test.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.test.tsx @@ -155,6 +155,15 @@ vi.mock("antd", () => { const Button = ({ children, htmlType, ...props }: { children?: any; htmlType?: string }) => React.createElement("button", { ...props, type: htmlType ?? props.type }, children); + const Typography = ({ children, ...props }: { children?: any }) => + React.createElement("div", props, children); + Typography.Text = ({ children, ...props }: { children?: any }) => + React.createElement("span", props, children); + Typography.Paragraph = ({ children, ...props }: { children?: any }) => + React.createElement("p", props, children); + Typography.Title = ({ children, ...props }: { children?: any }) => + React.createElement("h1", props, children); + return { Button, Form, @@ -171,6 +180,7 @@ vi.mock("antd", () => { Switch, Tag, Tooltip, + Typography, }; }); From 9380940cedaecdac720524deb2a37933730a1339 Mon Sep 17 00:00:00 2001 From: milan-berri Date: Sat, 9 May 2026 22:10:54 +0300 Subject: [PATCH 40/85] fix(mcp): forward extra_headers for OpenAPI MCP tools (#27383) * fix(mcp): forward extra_headers for OpenAPI MCP tools OpenAPI-generated tools only applied static closure headers and BYOK Authorization via ContextVar. Copy MCPServer.extra_headers from the incoming MCP request into _request_extra_headers (set in server.py before local tool dispatch), merge in openapi_to_mcp_generator via a small helper. OAuth2 M2M: do not forward caller Authorization from raw_headers (same rule as _prepare_mcp_server_headers for managed MCP). Adds TestRequestExtraHeaders and clarifies mcp_server_manager registration comment. Fixes #26794 Co-authored-by: Cursor * refactor(mcp): access has_client_credentials on MCPServer directly Greptile: getattr default was redundant; property exists on MCPServer and mcp_server is non-None inside the extra_headers forwarding block. Co-authored-by: Cursor --------- Co-authored-by: Cursor Co-authored-by: Mateo Wang --- .../mcp_server/mcp_server_manager.py | 3 +- .../mcp_server/openapi_to_mcp_generator.py | 56 ++++- .../proxy/_experimental/mcp_server/server.py | 30 +++ .../test_openapi_to_mcp_generator.py | 196 ++++++++++++++++++ 4 files changed, 276 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 55d5e4409e8..6ad731e7113 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -506,7 +506,8 @@ class MCPServerManager: # Add any static headers from server config. # # Note: `extra_headers` on MCPServer is a List[str] of header names to forward - # from the client request (not available in this OpenAPI tool generation step). + # from each client MCP request; values are applied at call time via + # `_request_extra_headers` in server.py (not baked in here). # `static_headers` is a dict of concrete headers to always send. headers = ( merge_mcp_headers( diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index 718435cce6f..271517bb1e6 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -55,6 +55,13 @@ _request_auth_header: contextvars.ContextVar[Optional[str]] = contextvars.Contex "_request_auth_header", default=None ) +# Per-request extra headers forwarded from the client request. +# Populated from MCPServer.extra_headers names matched against raw request +# headers in server.py before dispatching to a local/OpenAPI tool handler. +_request_extra_headers: contextvars.ContextVar[Optional[Dict[str, str]]] = ( + contextvars.ContextVar("_request_extra_headers", default=None) +) + def _sanitize_path_parameter_value(param_value: Any, param_name: str) -> str: """Ensure path params cannot introduce directory traversal.""" @@ -297,6 +304,46 @@ def build_input_schema(operation: Dict[str, Any]) -> Dict[str, Any]: } +def _merge_openapi_tool_request_headers( + static_headers: Dict[str, str] +) -> Dict[str, str]: + """Merge static closure headers with per-request ContextVar overrides. + + Precedence (highest to lowest): + 1. ``_request_auth_header`` — BYOK override of ``Authorization`` + 2. ``static_headers`` — operator-configured headers baked into the + tool closure at registration time + 3. ``_request_extra_headers`` — per-request headers forwarded from + the MCP caller (allowlisted by ``MCPServer.extra_headers``) + + This matches the existing MCP invariant in + :func:`litellm.proxy._experimental.mcp_server.utils.merge_mcp_headers` + and the managed MCP path, where ``static_headers`` always wins over + caller-forwarded headers. Keeping the same precedence here prevents an + authenticated caller from overriding an operator-configured value + (e.g. a tenant id or upstream API key) by sending the same header name. + + Header names are compared case-insensitively so different casing cannot + bypass the precedence rules. + """ + request_extra = _request_extra_headers.get() or {} + static = static_headers or {} + + static_lower_names = {k.lower() for k in static} + effective_headers: Dict[str, str] = { + k: v for k, v in request_extra.items() if k.lower() not in static_lower_names + } + effective_headers.update(static) + + override_auth = _request_auth_header.get() + if override_auth: + for existing in [k for k in effective_headers if k.lower() == "authorization"]: + del effective_headers[existing] + effective_headers["Authorization"] = override_auth + + return effective_headers + + def create_tool_function( path: str, method: str, @@ -334,14 +381,7 @@ def create_tool_function( The function safely handles parameter names that aren't valid Python identifiers by using **kwargs instead of named parameters. """ - # Allow per-request auth override (e.g. BYOK credential set via ContextVar). - # The ContextVar holds the full Authorization header value, including the - # correct prefix (Bearer / ApiKey / Basic) formatted by the caller in - # server.py based on the server's configured auth_type. - effective_headers = dict(headers) - override_auth = _request_auth_header.get() - if override_auth: - effective_headers["Authorization"] = override_auth + effective_headers = _merge_openapi_tool_request_headers(headers) # Build URL from base_url and path url = base_url + path diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 54d9bbe6e28..276a6e8a3bb 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -158,6 +158,7 @@ if MCP_AVAILABLE: ) from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( _request_auth_header, + _request_extra_headers, ) from litellm.proxy._experimental.mcp_server.sse_transport import SseServerTransport from litellm.proxy._experimental.mcp_server.tool_registry import ( @@ -2195,11 +2196,40 @@ if MCP_AVAILABLE: auth_header_value = f"Basic {mcp_auth_header}" else: auth_header_value = f"Bearer {mcp_auth_header}" + + # Forward named client headers to OpenAPI tool upstream requests. + # MCPServer.extra_headers lists header names to copy from raw_headers. + # OAuth2 M2M: never take Authorization from the caller (matches + # _prepare_mcp_server_headers for managed MCP). + forwarded_headers: Optional[Dict[str, str]] = None + if mcp_server and mcp_server.extra_headers and raw_headers: + normalized_raw = { + str(k).lower(): v + for k, v in raw_headers.items() + if isinstance(k, str) + } + skip_caller_authorization = bool(mcp_server.has_client_credentials) + for header_name in mcp_server.extra_headers: + if not isinstance(header_name, str): + continue + if ( + skip_caller_authorization + and header_name.lower() == "authorization" + ): + continue + value = normalized_raw.get(header_name.lower()) + if value is not None: + if forwarded_headers is None: + forwarded_headers = {} + forwarded_headers[header_name] = value + _auth_token = _request_auth_header.set(auth_header_value) + _extra_token = _request_extra_headers.set(forwarded_headers) try: local_content = await _handle_local_mcp_tool(name, arguments) finally: _request_auth_header.reset(_auth_token) + _request_extra_headers.reset(_extra_token) response = CallToolResult(content=cast(Any, local_content), isError=False) # Try managed MCP server tool (pass the full prefixed name) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py index 957dea22f3c..39f3c767220 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py @@ -15,6 +15,8 @@ from unittest.mock import AsyncMock, patch import pytest from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + _request_auth_header, + _request_extra_headers, _resolve_param_list, _resolve_ref, build_input_schema, @@ -1011,3 +1013,197 @@ class TestRegisterToolsFromOpenAPI: assert re.match( r"^[a-zA-Z0-9_-]+$", name ), f"fallback tool name {name!r} not sanitized" + + +class TestRequestExtraHeaders: + """Tests for _request_extra_headers ContextVar forwarding in tool_function.""" + + @pytest.mark.asyncio + async def test_extra_headers_forwarded_to_upstream(self): + """Extra headers set via ContextVar are included in the upstream request.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + token = _request_extra_headers.set({"X-TOKEN": "secret-value"}) + try: + result = await func() + finally: + _request_extra_headers.reset(token) + + assert result == "ok" + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent.get("X-TOKEN") == "secret-value" + + @pytest.mark.asyncio + async def test_no_extra_headers_by_default(self): + """Without setting _request_extra_headers, no extra headers are injected.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + headers={"X-Static": "static-value"}, + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + result = await func() + + assert result == "ok" + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent == {"X-Static": "static-value"} + assert "X-TOKEN" not in headers_sent + + @pytest.mark.asyncio + async def test_extra_headers_merged_with_static_headers(self): + """Forwarded headers are passed through alongside non-conflicting static headers.""" + operation = {} + func = create_tool_function( + path="/data", + method="post", + operation=operation, + base_url="https://api.example.com", + headers={"X-Static": "static-value"}, + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("post", "created") + mock_client.return_value = async_client + + token = _request_extra_headers.set({"X-TOKEN": "dynamic-value"}) + try: + result = await func() + finally: + _request_extra_headers.reset(token) + + assert result == "created" + call_args = async_client.post.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent.get("X-Static") == "static-value" + assert headers_sent.get("X-TOKEN") == "dynamic-value" + + @pytest.mark.asyncio + async def test_static_headers_win_over_forwarded_on_conflict(self): + """Static (operator) headers must override forwarded (caller) headers on name conflict.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + headers={"X-Tenant": "operator-tenant"}, + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + token = _request_extra_headers.set({"X-Tenant": "caller-spoofed"}) + try: + result = await func() + finally: + _request_extra_headers.reset(token) + + assert result == "ok" + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent.get("X-Tenant") == "operator-tenant" + assert "caller-spoofed" not in headers_sent.values() + + @pytest.mark.asyncio + async def test_static_headers_win_case_insensitively(self): + """Forwarded header with different casing must not bypass the static-wins rule.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + headers={"X-Tenant": "operator-tenant"}, + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + token = _request_extra_headers.set({"x-tenant": "caller-spoofed"}) + try: + result = await func() + finally: + _request_extra_headers.reset(token) + + assert result == "ok" + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent.get("X-Tenant") == "operator-tenant" + assert "x-tenant" not in headers_sent + assert "caller-spoofed" not in headers_sent.values() + + @pytest.mark.asyncio + async def test_auth_header_still_overrides_extra_headers(self): + """_request_auth_header takes precedence for Authorization over extra headers.""" + operation = {} + func = create_tool_function( + path="/secure", + method="get", + operation=operation, + base_url="https://api.example.com", + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "secure-data") + mock_client.return_value = async_client + + extra_token = _request_extra_headers.set( + {"Authorization": "Bearer extra", "X-TOKEN": "token-value"} + ) + auth_token = _request_auth_header.set("Bearer byok-credential") + try: + result = await func() + finally: + _request_auth_header.reset(auth_token) + _request_extra_headers.reset(extra_token) + + assert result == "secure-data" + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent.get("Authorization") == "Bearer byok-credential" + assert headers_sent.get("X-TOKEN") == "token-value" + + @pytest.mark.asyncio + async def test_extra_headers_not_leaked_between_calls(self): + """After resetting the ContextVar, subsequent calls do not see the headers.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + token = _request_extra_headers.set({"X-TOKEN": "first-call"}) + _request_extra_headers.reset(token) + + await func() + + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert "X-TOKEN" not in headers_sent From b834817785d0ce163d2422cdca452e7fff16574d Mon Sep 17 00:00:00 2001 From: Michael-RZ-Berri Date: Sat, 9 May 2026 12:32:16 -0700 Subject: [PATCH 41/85] [Feat] Add endpoint for bulk key updates for team (#26468) Squash-merged by litellm-agent from Michael-RZ-Berri's PR. --- litellm/proxy/_types.py | 2 + litellm/proxy/auth/route_checks.py | 2 + .../key_management_endpoints.py | 261 +++++++- .../key_management_endpoints.py | 80 ++- .../test_key_management_endpoints.py | 591 +++++++++++++++++- 5 files changed, 905 insertions(+), 31 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d58612d5054..29178d23059 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -239,6 +239,7 @@ class KeyManagementRoutes(str, enum.Enum): KEY_BLOCK = "/key/block" KEY_UNBLOCK = "/key/unblock" KEY_BULK_UPDATE = "/key/bulk_update" + TEAM_KEY_BULK_UPDATE = "/team/key/bulk_update" KEY_RESET_SPEND = "/key/{key_id}/reset_spend" # info and health routes @@ -540,6 +541,7 @@ class LiteLLMRoutes(enum.Enum): KeyManagementRoutes.KEY_BLOCK.value, KeyManagementRoutes.KEY_UNBLOCK.value, KeyManagementRoutes.KEY_BULK_UPDATE.value, + KeyManagementRoutes.TEAM_KEY_BULK_UPDATE.value, KeyManagementRoutes.TEAM_DAILY_ACTIVITY.value, KeyManagementRoutes.SPEND_LOGS.value, KeyManagementRoutes.KEY_RESET_SPEND.value, diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 8bcfbb67539..a9c36dfc512 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -50,6 +50,7 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES = frozenset( KeyManagementRoutes.KEY_BLOCK.value, KeyManagementRoutes.KEY_UNBLOCK.value, KeyManagementRoutes.KEY_BULK_UPDATE.value, + KeyManagementRoutes.TEAM_KEY_BULK_UPDATE.value, ] ) @@ -671,6 +672,7 @@ class RouteChecks: "/key/service-account/generate", "/key/block", "/key/unblock", + "/team/key/bulk_update", ] ) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index b112af1fe20..4439a55c1c6 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -88,8 +88,8 @@ from litellm.router import Router from litellm.secret_managers.main import get_secret from litellm.types.proxy.management_endpoints.key_management_endpoints import ( BulkUpdateKeyRequest, - BulkUpdateKeyRequestItem, BulkUpdateKeyResponse, + BulkUpdateTeamKeysRequest, FailedKeyUpdate, SuccessfulKeyUpdate, ) @@ -1881,7 +1881,7 @@ async def _get_and_validate_existing_key( async def _process_single_key_update( - key_update_item: BulkUpdateKeyRequestItem, + update_key_request: UpdateKeyRequest, user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: Optional[str], prisma_client: Optional[PrismaClient], @@ -1889,6 +1889,7 @@ async def _process_single_key_update( proxy_logging_obj: Any, llm_router: Optional[Router], user_custom_key_update: Optional[Callable] = None, + existing_key_row: Optional[LiteLLM_VerificationToken] = None, ) -> Dict[str, Any]: """ Process a single key update with all validations and checks. @@ -1897,13 +1898,14 @@ async def _process_single_key_update( including validation, permission checks, team checks, and database updates. Args: - key_update_item: The key update request item + update_key_request: Fully-constructed UpdateKeyRequest for the target key user_api_key_dict: The authenticated user's API key info litellm_changed_by: Optional header for tracking who made the change prisma_client: Prisma client instance user_api_key_cache: User API key cache proxy_logging_obj: Proxy logging object llm_router: LLM router instance + existing_key_row: Optional pre-fetched key row to avoid redundant lookups Returns: Dict containing the updated key information @@ -1912,13 +1914,14 @@ async def _process_single_key_update( HTTPException: For various validation and permission errors """ # Validate max_budget - _validate_max_budget(key_update_item.max_budget) + _validate_max_budget(update_key_request.max_budget) # Get and validate existing key - existing_key_row = await _get_and_validate_existing_key( - token=key_update_item.key, - prisma_client=prisma_client, - ) + if existing_key_row is None: + existing_key_row = await _get_and_validate_existing_key( + token=update_key_request.key, + prisma_client=prisma_client, + ) # Check team member permissions if prisma_client is not None: @@ -1930,15 +1933,6 @@ async def _process_single_key_update( user_api_key_cache=user_api_key_cache, ) - # Create UpdateKeyRequest from BulkUpdateKeyRequestItem - update_key_request = UpdateKeyRequest( - key=key_update_item.key, - budget_id=key_update_item.budget_id, - max_budget=key_update_item.max_budget, - team_id=key_update_item.team_id, - tags=key_update_item.tags, - ) - # Custom key update hook if user_custom_key_update is not None: if inspect.iscoroutinefunction(user_custom_key_update): @@ -2003,12 +1997,12 @@ async def _process_single_key_update( detail={"error": "Database not connected"}, ) - _data = {**non_default_values, "token": key_update_item.key} - response = await prisma_client.update_data(token=key_update_item.key, data=_data) + _data = {**non_default_values, "token": update_key_request.key} + response = await prisma_client.update_data(token=update_key_request.key, data=_data) # Delete cache await _delete_cache_key_object( - hashed_token=_hash_token_if_needed(key_update_item.key), + hashed_token=_hash_token_if_needed(update_key_request.key), user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) @@ -2598,9 +2592,15 @@ async def bulk_update_keys( for key_update_item in data.keys: try: - # Process single key update using reusable function + update_key_request = UpdateKeyRequest( + key=key_update_item.key, + budget_id=key_update_item.budget_id, + max_budget=key_update_item.max_budget, + team_id=key_update_item.team_id, + tags=key_update_item.tags, + ) updated_key_info = await _process_single_key_update( - key_update_item=key_update_item, + update_key_request=update_key_request, user_api_key_dict=user_api_key_dict, litellm_changed_by=litellm_changed_by, prisma_client=prisma_client, @@ -2665,6 +2665,223 @@ async def bulk_update_keys( ) +def _build_failed_team_key_update( + token: str, + exception: Exception, + existing_key_row: Optional[LiteLLM_VerificationToken], +) -> FailedKeyUpdate: + """Normalize an exception from the per-key update loop into a FailedKeyUpdate.""" + if isinstance(exception, HTTPException): + detail = exception.detail + if isinstance(detail, dict): + error_message = detail.get("error", str(exception)) + else: + error_message = str(detail) + elif isinstance(exception, ProxyException): + error_message = exception.message + else: + error_message = str(exception) + + key_info: Optional[Dict[str, Any]] = None + if existing_key_row is not None: + if hasattr(existing_key_row, "model_dump"): + key_info = existing_key_row.model_dump() + elif hasattr(existing_key_row, "dict"): + key_info = existing_key_row.dict() + if key_info: + key_info.pop("token", None) + + return FailedKeyUpdate(key=token, key_info=key_info, failed_reason=error_message) + + +@router.post( + "/team/key/bulk_update", + tags=["key management"], + dependencies=[Depends(user_api_key_auth)], + response_model=BulkUpdateKeyResponse, +) +@management_endpoint_wrapper +async def bulk_update_team_keys( + data: BulkUpdateTeamKeysRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + litellm_changed_by: Optional[str] = Header( + None, + description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + ), +): + """ + Apply one update payload to many keys inside a single team. + + Pass `team_id` plus either `key_ids` or `all_keys_in_team=True`. The + `update_fields` payload is broadcast to every selected key. Per-key + failures are returned in `failed_updates` rather than aborting the batch. + + Callable by proxy admins, or by team admins with `KEY_UPDATE` permission. + """ + from litellm.proxy.proxy_server import ( + llm_router, + prisma_client, + proxy_logging_obj, + user_api_key_cache, + user_custom_key_update, + ) + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": "Database not connected"}, + ) + + if not data.team_id: + raise HTTPException( + status_code=400, + detail={"error": "team_id is required"}, + ) + + MAX_BATCH_SIZE = 500 + if data.key_ids is not None and len(data.key_ids) > MAX_BATCH_SIZE: + raise HTTPException( + status_code=400, + detail={ + "error": f"Maximum {MAX_BATCH_SIZE} keys can be updated at once. Found {len(data.key_ids)} key_ids." + }, + ) + + if data.all_keys_in_team: + # "all" excludes blocked/expired — bulk refresh shouldn't revive a key an admin disabled. + # `blocked` is Boolean? with no default; `/key/generate` writes NULL. Prisma's `NOT` + # excludes NULLs, so explicitly OR `false` with `null` to include them. + now = datetime.now(timezone.utc) + existing_keys = await prisma_client.db.litellm_verificationtoken.find_many( + where={ + "team_id": data.team_id, + "AND": [ + {"OR": [{"blocked": False}, {"blocked": None}]}, + {"OR": [{"expires": None}, {"expires": {"gt": now}}]}, + ], + }, + order={"token": "asc"}, + take=MAX_BATCH_SIZE + 1, + ) + if len(existing_keys) > MAX_BATCH_SIZE: + raise HTTPException( + status_code=400, + detail={ + "error": f"Team {data.team_id} has more than {MAX_BATCH_SIZE} keys. Use `key_ids` to update in batches of {MAX_BATCH_SIZE}." + }, + ) + requested_tokens = [row.token for row in existing_keys] + else: + if data.key_ids is None or len(data.key_ids) == 0: + raise HTTPException( + status_code=400, + detail={ + "error": "key_ids must be provided when all_keys_in_team is False" + }, + ) + # Dedupe by hashed form — duplicates collapse to one update. + requested_tokens = [] + hashed_key_ids = [] + seen_hashes = set() + for k in data.key_ids: + h = _hash_token_if_needed(k) + if h in seen_hashes: + continue + seen_hashes.add(h) + requested_tokens.append(k) + hashed_key_ids.append(h) + existing_keys = await prisma_client.db.litellm_verificationtoken.find_many( + where={"team_id": data.team_id, "token": {"in": hashed_key_ids}} + ) + + # Anchor membership check on data.team_id (not existing_keys[0]); empty result must still gate non-admins. + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value: + auth_anchor = ( + existing_keys[0] + if existing_keys + else LiteLLM_VerificationToken( + token="__team_scope_auth_check__", + team_id=data.team_id, + models=[], + ) + ) + await TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint( + user_api_key_dict=user_api_key_dict, + route=KeyManagementRoutes.KEY_UPDATE, + prisma_client=prisma_client, + existing_key_row=auth_anchor, + user_api_key_cache=user_api_key_cache, + ) + + # Block metadata.allowed_passthrough_routes for non-admins — the runtime + # route checker reads it from key/team metadata to grant passthrough. + _check_passthrough_routes_caller_permission( + data=data.update_fields, user_api_key_dict=user_api_key_dict + ) + + if not requested_tokens: + raise HTTPException( + status_code=404, + detail={"error": f"No keys found for team {data.team_id}"}, + ) + + existing_by_token = {row.token: row for row in existing_keys} + update_field_dict = data.update_fields.model_dump(exclude_unset=True) + + successful_updates: List[SuccessfulKeyUpdate] = [] + failed_updates: List[FailedKeyUpdate] = [] + + for token in requested_tokens: + db_token = _hash_token_if_needed(token) + try: + if db_token not in existing_by_token: + raise HTTPException( + status_code=404, + detail={"error": f"Key not found in team {data.team_id}"}, + ) + + # team_id from validated scope, never user payload — drives _check_team_key_limits. + update_key_request = UpdateKeyRequest( + key=token, + team_id=data.team_id, + **update_field_dict, + ) + updated_key_info = await _process_single_key_update( + update_key_request=update_key_request, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + user_custom_key_update=user_custom_key_update, + existing_key_row=existing_by_token[db_token], + ) + + successful_updates.append( + SuccessfulKeyUpdate(key=token, key_info=updated_key_info) + ) + + except Exception as e: + # Log the hashed prefix — `token` may be a raw sk-... and ERROR logs persist. + verbose_proxy_logger.exception( + f"Failed to update key {db_token[:12]}... in team {data.team_id}: {e}" + ) + failed_updates.append( + _build_failed_team_key_update( + token=token, + exception=e, + existing_key_row=existing_by_token.get(db_token), + ) + ) + + return BulkUpdateKeyResponse( + total_requested=len(requested_tokens), + successful_updates=successful_updates, + failed_updates=failed_updates, + ) + + async def validate_key_team_change( key: LiteLLM_VerificationToken, team: LiteLLM_TeamTable, diff --git a/litellm/types/proxy/management_endpoints/key_management_endpoints.py b/litellm/types/proxy/management_endpoints/key_management_endpoints.py index b1d25455d18..d214cdb4f5d 100644 --- a/litellm/types/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/key_management_endpoints.py @@ -1,6 +1,7 @@ -from typing import Any, Dict, List, Optional +from datetime import datetime +from typing import Any, Dict, List, Literal, Optional -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, model_validator class BulkUpdateKeyRequestItem(BaseModel): @@ -40,3 +41,78 @@ class BulkUpdateKeyResponse(BaseModel): total_requested: int successful_updates: List[SuccessfulKeyUpdate] failed_updates: List[FailedKeyUpdate] + + +class KeyUpdateFields(BaseModel): + """Allowlist of bulk-broadcastable fields for /team/key/bulk_update; `extra="forbid"` blocks RBAC/ownership/scope mutations even by team admins.""" + + model_config = ConfigDict(extra="forbid", protected_namespaces=()) + + # Budgets + max_budget: Optional[float] = None + budget_id: Optional[str] = None + budget_duration: Optional[str] = None + budget_limits: Optional[List[Any]] = None + model_max_budget: Optional[Dict[str, Any]] = None + + # Rate limits + tpm_limit: Optional[int] = None + rpm_limit: Optional[int] = None + model_tpm_limit: Optional[Dict[str, Any]] = None + model_rpm_limit: Optional[Dict[str, Any]] = None + max_parallel_requests: Optional[int] = None + rpm_limit_type: Optional[ + Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] + ] = None + tpm_limit_type: Optional[ + Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] + ] = None + + # Temporary budget grants (auto-expire). `spend` deliberately omitted — bulk-zeroing it bypasses budget enforcement; admin-only via /key/update. + temp_budget_increase: Optional[float] = None + temp_budget_expiry: Optional[datetime] = None + + # Expiry + duration: Optional[str] = None + + # Operational metadata + tags: Optional[List[str]] = None + metadata: Optional[Dict[str, Any]] = None + + @model_validator(mode="after") + def validate_temp_budget(self) -> "KeyUpdateFields": + if self.temp_budget_increase is not None or self.temp_budget_expiry is not None: + if self.temp_budget_increase is None or self.temp_budget_expiry is None: + raise ValueError( + "temp_budget_increase and temp_budget_expiry must be set together" + ) + return self + + @model_validator(mode="after") + def require_at_least_one_field(self) -> "KeyUpdateFields": + # Reject empty payload — would iterate every key with no-op writes. + if not self.model_fields_set: + raise ValueError("update_fields must specify at least one field to update.") + return self + + +class BulkUpdateTeamKeysRequest(BaseModel): + """Apply one update payload to many keys inside a team; provide either `key_ids` or `all_keys_in_team=True`.""" + + team_id: str + key_ids: Optional[List[str]] = None + all_keys_in_team: bool = False + update_fields: KeyUpdateFields + + @model_validator(mode="after") + def validate_selection(self) -> "BulkUpdateTeamKeysRequest": + has_key_ids = self.key_ids is not None and len(self.key_ids) > 0 + if has_key_ids and self.all_keys_in_team: + raise ValueError( + "Provide either `key_ids` or `all_keys_in_team=True`, not both." + ) + if not has_key_ids and not self.all_keys_in_team: + raise ValueError( + "Must provide either `key_ids` (non-empty) or `all_keys_in_team=True`." + ) + return self diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index b292e8d0cae..66716400c4f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -5689,7 +5689,7 @@ async def test_process_single_key_update(): "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" ): # Create update request - key_update_item = BulkUpdateKeyRequestItem( + update_key_request = UpdateKeyRequest( key="test-key-123", max_budget=100.0, tags=["production"], @@ -5703,7 +5703,7 @@ async def test_process_single_key_update(): # Call the function result = await _process_single_key_update( - key_update_item=key_update_item, + update_key_request=update_key_request, user_api_key_dict=user_api_key_dict, litellm_changed_by=None, prisma_client=mock_prisma_client, @@ -9855,9 +9855,6 @@ async def test_process_single_key_update_cache_invalidation_with_token_hash(): from litellm.proxy.management_endpoints.key_management_endpoints import ( _process_single_key_update, ) - from litellm.types.proxy.management_endpoints.key_management_endpoints import ( - BulkUpdateKeyRequestItem, - ) token_hash = "abc123def456" @@ -9900,7 +9897,7 @@ async def test_process_single_key_update_cache_invalidation_with_token_hash(): new_callable=AsyncMock, ), ): - key_update_item = BulkUpdateKeyRequestItem( + update_key_request = UpdateKeyRequest( key=token_hash, max_budget=100.0, ) @@ -9912,7 +9909,7 @@ async def test_process_single_key_update_cache_invalidation_with_token_hash(): ) await _process_single_key_update( - key_update_item=key_update_item, + update_key_request=update_key_request, user_api_key_dict=user_api_key_dict, litellm_changed_by=None, prisma_client=mock_prisma_client, @@ -10019,3 +10016,583 @@ async def test_execute_virtual_key_regeneration_cache_invalidation_with_token_ha call_kwargs = mock_delete_cache.call_args.kwargs # The token hash should be passed as-is, NOT double-hashed assert call_kwargs["hashed_token"] == token_hash + + +# --------------------------------------------------------------------------- +# /team/key/bulk_update tests +# --------------------------------------------------------------------------- + + +_BULK_PKG = "litellm.proxy.management_endpoints.key_management_endpoints" + + +def _make_team_key(token: str, team_id: str = "team-abc") -> LiteLLM_VerificationToken: + return LiteLLM_VerificationToken( + token=token, + user_id="user-123", + models=[], + team_id=team_id, + max_budget=None, + ) + + +def _admin() -> UserAPIKeyAuth: + return UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin", user_id="admin" + ) + + +def _internal_user() -> UserAPIKeyAuth: + return UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-iu", user_id="iu" + ) + + +def _updated(payload): + m = MagicMock() + m.model_dump.return_value = payload + return m + + +def _setup_team_keys_mocks( + monkeypatch, + *, + find_many=None, + find_unique=None, + update_data=None, + hash_identity=True, +): + """Set up mocks for bulk_update_team_keys; returns mock_prisma.""" + mock_prisma = AsyncMock() + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[] if find_many is None else find_many + ) + if find_unique is not None: + mock_prisma.db.litellm_verificationtoken.find_unique = find_unique + if update_data is not None: + mock_prisma.update_data = update_data + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", MagicMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.user_custom_key_update", None) + monkeypatch.setattr( + f"{_BULK_PKG}.prepare_key_update_data", + AsyncMock(return_value={"max_budget": 50.0}), + ) + monkeypatch.setattr(f"{_BULK_PKG}._delete_cache_key_object", AsyncMock()) + monkeypatch.setattr( + f"{_BULK_PKG}.KeyManagementEventHooks.async_key_updated_hook", AsyncMock() + ) + monkeypatch.setattr(f"{_BULK_PKG}.get_team_object", AsyncMock(return_value=None)) + monkeypatch.setattr(f"{_BULK_PKG}._check_team_key_limits", AsyncMock()) + monkeypatch.setattr( + f"{_BULK_PKG}.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint", + AsyncMock(), + ) + if hash_identity: + # Tests use already-hashed tokens; the raw-sk regression opts out. + monkeypatch.setattr(f"{_BULK_PKG}._hash_token_if_needed", lambda token: token) + return mock_prisma + + +async def _call_as_admin(data): + from litellm.proxy.management_endpoints.key_management_endpoints import ( + bulk_update_team_keys, + ) + + return await bulk_update_team_keys( + data=data, user_api_key_dict=_admin(), litellm_changed_by=None + ) + + +# ---- happy paths ---------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_success_with_key_ids(monkeypatch): + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + keys = [_make_team_key("tok-a"), _make_team_key("tok-b")] + find_unique = AsyncMock(side_effect=keys) + mock = _setup_team_keys_mocks( + monkeypatch, + find_many=keys, + find_unique=find_unique, + update_data=AsyncMock( + side_effect=[{"data": _updated({"max_budget": 50.0})}] * 2 + ), + ) + + response = await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + key_ids=["tok-a", "tok-b"], + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + + assert len(response.successful_updates) == 2 + assert len(response.failed_updates) == 0 + where = mock.db.litellm_verificationtoken.find_many.await_args.kwargs["where"] + assert where["team_id"] == "team-abc" + assert where["token"] == {"in": ["tok-a", "tok-b"]} + find_unique.assert_not_called() + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_success_all_keys_in_team(monkeypatch): + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + keys = [_make_team_key(f"tok-{i}") for i in range(3)] + find_unique = AsyncMock(side_effect=keys) + mock = _setup_team_keys_mocks( + monkeypatch, + find_many=keys, + find_unique=find_unique, + update_data=AsyncMock( + side_effect=[{"data": _updated({"max_budget": 50.0})}] * 3 + ), + ) + + response = await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + all_keys_in_team=True, + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + + assert len(response.successful_updates) == 3 + where = mock.db.litellm_verificationtoken.find_many.await_args.kwargs["where"] + # `blocked` is Boolean? with no default → /key/generate writes NULL. Prisma's + # NOT excludes NULLs, so the filter has to OR `false` with `null` explicitly. + blocked_or, expires_or = where["AND"][0]["OR"], where["AND"][1]["OR"] + assert {"blocked": False} in blocked_or and {"blocked": None} in blocked_or + assert {"expires": None} in expires_or + assert any( + "gt" in c.get("expires", {}) + for c in expires_or + if isinstance(c.get("expires"), dict) + ) + find_unique.assert_not_called() + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_key_not_in_team(monkeypatch): + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + in_team = _make_team_key("tok-a") + _setup_team_keys_mocks( + monkeypatch, + find_many=[in_team], + find_unique=AsyncMock(return_value=in_team), + update_data=AsyncMock(return_value={"data": _updated({"max_budget": 50.0})}), + ) + + response = await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + key_ids=["tok-a", "tok-foreign"], + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + assert [u.key for u in response.successful_updates] == ["tok-a"] + assert [u.key for u in response.failed_updates] == ["tok-foreign"] + assert "not found in team" in response.failed_updates[0].failed_reason + + +# ---- error paths ---------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_batch_size_cap(monkeypatch): + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + _setup_team_keys_mocks( + monkeypatch, + find_many=[_make_team_key(f"tok-{i}") for i in range(501)], + ) + + with pytest.raises(HTTPException) as exc: + await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + all_keys_in_team=True, + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + assert exc.value.status_code == 400 + assert "more than 500" in exc.value.detail["error"] + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_empty_team_returns_404(monkeypatch): + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + _setup_team_keys_mocks(monkeypatch, find_many=[]) + with pytest.raises(HTTPException) as exc: + await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-empty", + all_keys_in_team=True, + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + assert exc.value.status_code == 404 + + +# ---- auth ----------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_team_member_with_permission(monkeypatch): + from litellm.proxy.management_endpoints.key_management_endpoints import ( + bulk_update_team_keys, + ) + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + key_a = _make_team_key("tok-a") + _setup_team_keys_mocks( + monkeypatch, + find_many=[key_a], + find_unique=AsyncMock(return_value=key_a), + update_data=AsyncMock(return_value={"data": _updated({"max_budget": 50.0})}), + ) + auth_check = AsyncMock() + monkeypatch.setattr( + f"{_BULK_PKG}.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint", + auth_check, + ) + + response = await bulk_update_team_keys( + data=BulkUpdateTeamKeysRequest( + team_id="team-abc", + all_keys_in_team=True, + update_fields=KeyUpdateFields(max_budget=50.0), + ), + user_api_key_dict=_internal_user(), + litellm_changed_by=None, + ) + assert len(response.successful_updates) == 1 + # Upfront check + per-key check inside _process_single_key_update + assert auth_check.await_count == 2 + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_team_member_no_permission(monkeypatch): + from litellm.proxy.management_endpoints.key_management_endpoints import ( + bulk_update_team_keys, + ) + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + mock = _setup_team_keys_mocks(monkeypatch, find_many=[_make_team_key("tok-a")]) + monkeypatch.setattr( + f"{_BULK_PKG}.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint", + AsyncMock( + side_effect=ProxyException( + message="not in team", + type="team_member_permission_error", + param="/key/update", + code=401, + ) + ), + ) + + with pytest.raises(ProxyException): + await bulk_update_team_keys( + data=BulkUpdateTeamKeysRequest( + team_id="team-abc", + all_keys_in_team=True, + update_fields=KeyUpdateFields(max_budget=1.0), + ), + user_api_key_dict=_internal_user(), + litellm_changed_by=None, + ) + mock.update_data.assert_not_called() + + +# ---- pydantic-layer validation ------------------------------------------- + + +def test_bulk_update_team_keys_request_validation(): + """Allowlist (extra='forbid'), empty-payload rejection, and selection XOR.""" + from pydantic import ValidationError + + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + forbidden = [ + "key", + "key_alias", + "team_id", + "allowed_routes", + "allowed_passthrough_routes", + "permissions", + "object_permission", + "access_group_ids", + "user_id", + "organization_id", + "blocked", + "key_type", + "models", + "config", + "router_settings", + "spend", + ] + for f in forbidden: + with pytest.raises(ValidationError, match=f): + KeyUpdateFields(**{f: True}) + + with pytest.raises(ValidationError, match="at least one"): + KeyUpdateFields() + + assert KeyUpdateFields(max_budget=50.0, tags=["x"]).max_budget == 50.0 + + valid = KeyUpdateFields(max_budget=10) + with pytest.raises(ValidationError): + BulkUpdateTeamKeysRequest( + team_id="t", key_ids=["k"], all_keys_in_team=True, update_fields=valid + ) + with pytest.raises(ValidationError): + BulkUpdateTeamKeysRequest(team_id="t", update_fields=valid) + + +# ---- security regressions ------------------------------------------------ + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_hashes_raw_sk_key_ids(monkeypatch): + """Regression: raw sk-... key_ids must be hashed before the find_many lookup.""" + from litellm.proxy._types import hash_token + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + raw_sk = "sk-rawkey1234567890" + hashed = hash_token(raw_sk) + row = LiteLLM_VerificationToken( + token=hashed, user_id="u", models=[], team_id="team-abc", max_budget=None + ) + mock = _setup_team_keys_mocks( + monkeypatch, + find_many=[row], + find_unique=AsyncMock(return_value=row), + update_data=AsyncMock(return_value={"data": _updated({"max_budget": 50.0})}), + hash_identity=False, + ) + + response = await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + key_ids=[raw_sk], + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + where = mock.db.litellm_verificationtoken.find_many.await_args.kwargs["where"] + assert where["token"] == {"in": [hashed]} + # Response reports the user-supplied form, not the hash. + assert response.successful_updates[0].key == raw_sk + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_auth_check_runs_when_no_keys_match(monkeypatch): + """Regression: non-admin with bogus key_ids must still hit the membership gate.""" + from litellm.proxy.management_endpoints.key_management_endpoints import ( + bulk_update_team_keys, + ) + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + mock = _setup_team_keys_mocks(monkeypatch, find_many=[]) + auth_check = AsyncMock( + side_effect=ProxyException( + message="not in team", + type="team_member_permission_error", + param="/key/update", + code=401, + ) + ) + monkeypatch.setattr( + f"{_BULK_PKG}.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint", + auth_check, + ) + + with pytest.raises(ProxyException): + await bulk_update_team_keys( + data=BulkUpdateTeamKeysRequest( + team_id="victim-team", + key_ids=["bogus-1", "bogus-2"], + update_fields=KeyUpdateFields(max_budget=1.0), + ), + user_api_key_dict=_internal_user(), + litellm_changed_by=None, + ) + # Anchored on data.team_id, not existing_keys[0]. + assert auth_check.await_args.kwargs["existing_key_row"].team_id == "victim-team" + mock.update_data.assert_not_called() + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_does_not_log_raw_sk_token_on_failure( + monkeypatch, caplog +): + """Regression: per-key failure must not log the raw sk-... (ERROR-level logs persist).""" + import logging + + from litellm.proxy._types import hash_token + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + raw_sk = "sk-supersecret1234567890" + row = LiteLLM_VerificationToken( + token=hash_token(raw_sk), + user_id="u", + models=[], + team_id="team-abc", + max_budget=None, + ) + _setup_team_keys_mocks( + monkeypatch, + find_many=[row], + update_data=AsyncMock(side_effect=RuntimeError("boom")), + hash_identity=False, + ) + + with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): + response = await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + key_ids=[raw_sk], + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + assert len(response.failed_updates) == 1 + log_text = "\n".join(r.getMessage() for r in caplog.records) + assert raw_sk not in log_text + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_propagates_team_id_to_per_key_request(monkeypatch): + """Regression: per-key UpdateKeyRequest carries data.team_id (gates _check_team_key_limits).""" + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + _setup_team_keys_mocks(monkeypatch, find_many=[_make_team_key("tok-a")]) + captured = [] + + async def fake_process(*, update_key_request, **kw): + captured.append(update_key_request) + return {"max_budget": update_key_request.max_budget} + + monkeypatch.setattr(f"{_BULK_PKG}._process_single_key_update", fake_process) + + await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + key_ids=["tok-a"], + update_fields=KeyUpdateFields( + tpm_limit=10_000, tpm_limit_type="guaranteed_throughput" + ), + ) + ) + assert captured[0].team_id == "team-abc" + assert captured[0].tpm_limit_type == "guaranteed_throughput" + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_dedupes_key_ids(monkeypatch): + """Duplicate key_ids collapse to a single update (no redundant DB writes, no inflated counts).""" + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + key_a = _make_team_key("tok-a") + update_data = AsyncMock(return_value={"data": _updated({"max_budget": 50.0})}) + _setup_team_keys_mocks( + monkeypatch, + find_many=[key_a], + find_unique=AsyncMock(return_value=key_a), + update_data=update_data, + ) + + response = await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + key_ids=["tok-a", "tok-a", "tok-a"], + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + + assert response.total_requested == 1 + assert len(response.successful_updates) == 1 + assert len(response.failed_updates) == 0 + update_data.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_blocks_metadata_allowed_passthrough_routes( + monkeypatch, +): + """Non-admin can't grant passthrough access by smuggling allowed_passthrough_routes through metadata.""" + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.key_management_endpoints import ( + bulk_update_team_keys, + ) + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + mock = _setup_team_keys_mocks(monkeypatch, find_many=[_make_team_key("tok-a")]) + + request = BulkUpdateTeamKeysRequest( + team_id="team-abc", + all_keys_in_team=True, + update_fields=KeyUpdateFields( + metadata={"allowed_passthrough_routes": ["/admin/*"]} + ), + ) + + with pytest.raises(HTTPException) as exc: + await bulk_update_team_keys( + data=request, + user_api_key_dict=_internal_user(), + litellm_changed_by=None, + ) + + assert exc.value.status_code == 403 + assert "allowed_passthrough_routes" in str(exc.value.detail) + mock.update_data.assert_not_called() From 2b4beae29a3025d5f6dc24cd42c76cbd713fc603 Mon Sep 17 00:00:00 2001 From: Noah Nistler <60981020+noahnistler@users.noreply.github.com> Date: Sat, 9 May 2026 15:14:40 -0500 Subject: [PATCH 42/85] Fix/shared health check polling (#26434) Squash-merged by litellm-agent from noahnistler's PR. --- .../shared_health_check_manager.py | 62 ++++++-- .../proxy/test_shared_health_check.py | 143 ++++++++++++++++-- 2 files changed, 180 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/health_check_utils/shared_health_check_manager.py b/litellm/proxy/health_check_utils/shared_health_check_manager.py index 5b8370fece8..ad58bc7e286 100644 --- a/litellm/proxy/health_check_utils/shared_health_check_manager.py +++ b/litellm/proxy/health_check_utils/shared_health_check_manager.py @@ -253,27 +253,63 @@ class SharedHealthCheckManager: # Always release the lock await self.release_health_check_lock() else: - # Lock not acquired, wait briefly and try to get cached results + # If Redis is not configured, skip polling — there is no cache + # to wait for. + if self.redis_cache is None: + return await perform_health_check( + model_list=model_list, + details=details, + max_concurrency=max_concurrency, + ) + + # Lock not acquired — poll for cached results until the lock + # holder finishes or the lock expires, rather than falling back + # to a redundant local health check after only 2 seconds. verbose_proxy_logger.debug( "Pod %s waiting for other pod to complete health check", self.pod_id ) - # Wait a bit for the other pod to complete - await asyncio.sleep(2) + poll_interval = 5 # seconds between cache checks + max_wait = self.lock_ttl # wait at most as long as the lock can live + elapsed = 0 - # Try to get cached results again - cached_results = await self.get_cached_health_check_results() - if cached_results is not None: - return ( - cached_results.get("healthy_endpoints", []), - cached_results.get("unhealthy_endpoints", []), - {}, - ) + while elapsed < max_wait: + await asyncio.sleep(poll_interval) + elapsed += poll_interval - # Still no cache, fall back to local health check + cached_results = await self.get_cached_health_check_results() + if cached_results is not None: + verbose_proxy_logger.info( + "Pod %s using cached health check results after waiting %ds", + self.pod_id, + elapsed, + ) + return ( + cached_results.get("healthy_endpoints", []), + cached_results.get("unhealthy_endpoints", []), + {}, + ) + + # Check if the lock is still held — if it was released without + # caching (e.g. the holder crashed), stop waiting early. + try: + lock_key = self.get_health_check_lock_key() + current_owner = await self.redis_cache.async_get_cache(lock_key) + if current_owner is None: + verbose_proxy_logger.debug( + "Pod %s detected lock released without cache, stopping wait", + self.pod_id, + ) + break + except Exception: + # Redis hiccup — continue polling rather than crashing out + pass + + # Exhausted wait — fall back to local health check verbose_proxy_logger.warning( - "Pod %s falling back to local health check (no cache available)", + "Pod %s falling back to local health check after waiting %ds (no cache available)", self.pod_id, + elapsed, ) return await perform_health_check( diff --git a/tests/test_litellm/proxy/test_shared_health_check.py b/tests/test_litellm/proxy/test_shared_health_check.py index 04099f2634b..1530d336085 100644 --- a/tests/test_litellm/proxy/test_shared_health_check.py +++ b/tests/test_litellm/proxy/test_shared_health_check.py @@ -322,13 +322,13 @@ class TestSharedHealthCheckManager: async def test_perform_shared_health_check_lock_failed_then_cache( self, shared_health_manager, mock_redis_cache ): - """Test performing shared health check when lock fails but cache becomes available""" + """Test performing shared health check when lock fails but cache becomes available during polling""" # First call: no cache, lock fails - # Second call: cache available + # Polling finds cache on first iteration mock_redis_cache.async_get_cache.side_effect = [ - None, # No cache initially + None, # No cache initially (get_cached_health_check_results) json.dumps( - { # Cache available after waiting + { # Cache available on first poll iteration "healthy_endpoints": [{"model": "cached-model"}], "unhealthy_endpoints": [], "healthy_count": 1, @@ -350,18 +350,68 @@ class TestSharedHealthCheckManager: ) ) - # Should wait and then get cached results - mock_sleep.assert_called_once_with(2) + # Should poll once (5s interval) and find cached results + mock_sleep.assert_called_once_with(5) assert healthy == [{"model": "cached-model"}] assert unhealthy == [] @pytest.mark.asyncio - async def test_perform_shared_health_check_fallback( + async def test_perform_shared_health_check_fallback(self, mock_redis_cache): + """Test performing shared health check with fallback to local health check""" + # Use short lock_ttl so the polling loop only runs 2 iterations + manager = SharedHealthCheckManager( + redis_cache=mock_redis_cache, + health_check_ttl=300, + lock_ttl=10, + ) + + # No cache ever, lock always held by another pod + mock_redis_cache.async_get_cache.side_effect = [ + None, # Initial cache check + None, # Iteration 1: cache check + "other_pod", # Iteration 1: lock check (still held) + None, # Iteration 2: cache check + "other_pod", # Iteration 2: lock check (still held) + ] + mock_redis_cache.async_set_cache.return_value = False # Lock acquisition fails + + model_list = [ + {"model_name": "test-model", "litellm_params": {"model": "test-model"}} + ] + expected_healthy = [{"model": "test-model", "status": "healthy"}] + expected_unhealthy = [] + + with ( + patch("asyncio.sleep") as mock_sleep, + patch( + "litellm.proxy.health_check_utils.shared_health_check_manager.perform_health_check" + ) as mock_perform, + ): + mock_perform.return_value = (expected_healthy, expected_unhealthy, {}) + + healthy, unhealthy, _ = await manager.perform_shared_health_check( + model_list, details=True + ) + + # Should poll twice (5s * 2 = 10s >= lock_ttl) then fall back + assert mock_sleep.call_count == 2 + mock_sleep.assert_called_with(5) + mock_perform.assert_called_once_with( + model_list=model_list, details=True, max_concurrency=None + ) + assert healthy == expected_healthy + assert unhealthy == expected_unhealthy + + @pytest.mark.asyncio + async def test_perform_shared_health_check_early_exit_orphaned_lock( self, shared_health_manager, mock_redis_cache ): - """Test performing shared health check with fallback to local health check""" - # No cache, lock fails, no cache after waiting - mock_redis_cache.async_get_cache.return_value = None + """Test that polling exits early when the lock disappears without a cache write (crash recovery)""" + mock_redis_cache.async_get_cache.side_effect = [ + None, # Initial cache check + None, # Iteration 1: cache check (still no cache) + None, # Iteration 1: lock check -> lock gone (holder crashed) + ] mock_redis_cache.async_set_cache.return_value = False # Lock acquisition fails model_list = [ @@ -384,8 +434,77 @@ class TestSharedHealthCheckManager: ) ) - # Should fall back to local health check - mock_sleep.assert_called_once_with(2) + # Should detect orphaned lock after 1 iteration and fall back immediately + mock_sleep.assert_called_once_with(5) + mock_perform.assert_called_once_with( + model_list=model_list, details=True, max_concurrency=None + ) + assert healthy == expected_healthy + assert unhealthy == expected_unhealthy + + @pytest.mark.asyncio + async def test_perform_shared_health_check_redis_error_during_polling( + self, shared_health_manager, mock_redis_cache + ): + """Test that a transient Redis error during lock polling doesn't crash the loop""" + cached_data = json.dumps( + { + "healthy_endpoints": [{"model": "cached-model"}], + "unhealthy_endpoints": [], + "healthy_count": 1, + "unhealthy_count": 0, + "timestamp": time.time() - 100, + } + ) + mock_redis_cache.async_get_cache.side_effect = [ + None, # Initial cache check + None, # Iteration 1: cache check + Exception("Redis connection lost"), # Iteration 1: lock check errors + cached_data, # Iteration 2: cache check -> found! + ] + mock_redis_cache.async_set_cache.return_value = False # Lock acquisition fails + + model_list = [ + {"model_name": "test-model", "litellm_params": {"model": "test-model"}} + ] + + with patch("asyncio.sleep") as mock_sleep: + healthy, unhealthy, _ = ( + await shared_health_manager.perform_shared_health_check( + model_list, details=True + ) + ) + + # Should survive the Redis error and find cache on iteration 2 + assert mock_sleep.call_count == 2 + assert healthy == [{"model": "cached-model"}] + assert unhealthy == [] + + @pytest.mark.asyncio + async def test_perform_shared_health_check_no_redis_skips_polling(self): + """Test that polling is skipped entirely when redis_cache is None""" + manager = SharedHealthCheckManager(redis_cache=None) + + model_list = [ + {"model_name": "test-model", "litellm_params": {"model": "test-model"}} + ] + expected_healthy = [{"model": "test-model", "status": "healthy"}] + expected_unhealthy = [] + + with ( + patch("asyncio.sleep") as mock_sleep, + patch( + "litellm.proxy.health_check_utils.shared_health_check_manager.perform_health_check" + ) as mock_perform, + ): + mock_perform.return_value = (expected_healthy, expected_unhealthy, {}) + + healthy, unhealthy, _ = await manager.perform_shared_health_check( + model_list, details=True + ) + + # Should NOT sleep at all — falls back to local health check immediately + mock_sleep.assert_not_called() mock_perform.assert_called_once_with( model_list=model_list, details=True, max_concurrency=None ) From 0f908e6885c31a390a393e1475b638e7382a162b Mon Sep 17 00:00:00 2001 From: Tai An Date: Sat, 9 May 2026 13:23:15 -0700 Subject: [PATCH 43/85] fix(proxy): resolve provider from deployment for multi-provider defaultconfig (#27516) (#27517) Squash-merged by litellm-agent from Anai-Guo's PR. --- litellm/proxy/litellm_pre_call_utils.py | 55 +++++- .../proxy/test_litellm_pre_call_utils.py | 172 ++++++++++++++++++ 2 files changed, 226 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index a63613c5836..9d7d6f476ac 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1731,6 +1731,7 @@ async def add_litellm_data_to_request( # noqa: PLR0915 data=data, user_api_key_dict=user_api_key_dict, pre_alias_model_name=_pre_alias_model, + llm_router=llm_router, ) ## ENFORCED PARAMS CHECK @@ -1864,6 +1865,7 @@ def _apply_credential_overrides_from_model_config( data: dict, user_api_key_dict: UserAPIKeyAuth, pre_alias_model_name: Optional[str] = None, + llm_router: Optional[Router] = None, ) -> None: """ Walk the model_config precedence chain in team/project metadata. @@ -1899,10 +1901,19 @@ def _apply_credential_overrides_from_model_config( if not project_model_config and not team_model_config: return - # Extract provider hint from model name (e.g. "azure/gpt-4" -> "azure") + # Extract provider hint from model name (e.g. "azure/gpt-4" -> "azure"). + # When the user-facing name has no provider prefix, fall back to the + # deployment's litellm_params so multi-provider defaultconfig entries + # don't silently match the first dict key (#27516). provider: Optional[str] = None if "/" in model_name: provider = model_name.split("/", 1)[0] + elif llm_router is not None: + provider = _resolve_provider_from_deployment( + llm_router=llm_router, + model_name=model_name, + pre_alias_model_name=pre_alias_model_name, + ) credential_name = _resolve_credential_from_model_config( model_name=model_name, @@ -1938,6 +1949,48 @@ def _apply_credential_overrides_from_model_config( ) +def _resolve_provider_from_deployment( + llm_router: Router, + model_name: str, + pre_alias_model_name: Optional[str] = None, +) -> Optional[str]: + """ + Resolve a provider hint from the deployment's litellm_params when the + user-facing model name has no provider prefix. + + Tries the post-alias name first (the resolved model group), then the + pre-alias name. Returns None if no deployment is found or the deployment + has no usable provider info. + """ + candidates = [model_name] + if pre_alias_model_name and pre_alias_model_name != model_name: + candidates.append(pre_alias_model_name) + + for name in candidates: + try: + deployment = llm_router.get_deployment_by_model_group_name( + model_group_name=name + ) + except Exception: + deployment = None + if deployment is None: + continue + + litellm_params = getattr(deployment, "litellm_params", None) + if litellm_params is None: + continue + + custom_provider = getattr(litellm_params, "custom_llm_provider", None) + if custom_provider: + return custom_provider + + deployment_model = getattr(litellm_params, "model", "") or "" + if "/" in deployment_model: + return deployment_model.split("/", 1)[0] + + return None + + def _resolve_credential_from_model_config( model_name: str, project_model_config: Optional[dict], diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 92611431a15..b803dfb709a 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -21,6 +21,7 @@ from litellm.proxy.litellm_pre_call_utils import ( _get_enforced_params, _get_metadata_variable_name, _resolve_credential_from_model_config, + _resolve_provider_from_deployment, _update_model_if_key_alias_exists, add_guardrails_from_policy_engine, add_litellm_data_to_request, @@ -4043,3 +4044,174 @@ def test_get_guardrail_from_metadata_reads_litellm_metadata_when_no_metadata(): assert result == [ "my-guardrail" ], f"Expected guardrails from litellm_metadata fallback, got: {result}" + + +# ============================================================================ +# Tests for #27516: provider hint resolution from deployment when the +# user-facing model name has no provider prefix. +# ============================================================================ + + +def test_resolve_provider_from_deployment_uses_litellm_params_model(): + """When custom_llm_provider is unset, fall back to the prefix of model.""" + router = MagicMock() + deployment = MagicMock() + deployment.litellm_params.model = "bedrock/us.anthropic.claude-sonnet-4-6" + deployment.litellm_params.custom_llm_provider = None + router.get_deployment_by_model_group_name.return_value = deployment + + assert ( + _resolve_provider_from_deployment(router, "claude-sonnet-4.6") == "bedrock" + ) + + +def test_resolve_provider_from_deployment_prefers_custom_llm_provider(): + """Explicit custom_llm_provider on the deployment wins over model prefix.""" + router = MagicMock() + deployment = MagicMock() + deployment.litellm_params.model = "us.anthropic.claude-sonnet-4-6" + deployment.litellm_params.custom_llm_provider = "bedrock" + router.get_deployment_by_model_group_name.return_value = deployment + + assert ( + _resolve_provider_from_deployment(router, "claude-sonnet-4.6") == "bedrock" + ) + + +def test_resolve_provider_from_deployment_no_match(): + """No deployment for the model group -> None.""" + router = MagicMock() + router.get_deployment_by_model_group_name.return_value = None + assert _resolve_provider_from_deployment(router, "unknown-model") is None + + +def test_resolve_provider_from_deployment_router_raises(): + """Router exceptions must not propagate — fall back to None.""" + router = MagicMock() + router.get_deployment_by_model_group_name.side_effect = RuntimeError("boom") + assert _resolve_provider_from_deployment(router, "claude-sonnet-4.6") is None + + +def test_resolve_provider_from_deployment_falls_back_to_pre_alias(): + """If post-alias lookup fails, the pre-alias name is also tried.""" + router = MagicMock() + deployment = MagicMock() + deployment.litellm_params.model = "bedrock/anthropic.claude-sonnet-4-6" + deployment.litellm_params.custom_llm_provider = None + + def lookup(model_group_name): + if model_group_name == "pre-alias-name": + return deployment + return None + + router.get_deployment_by_model_group_name.side_effect = lookup + + result = _resolve_provider_from_deployment( + router, "post-alias-name", pre_alias_model_name="pre-alias-name" + ) + assert result == "bedrock" + + +def test_apply_overrides_multi_provider_default_picks_correct_provider( + setup_test_credentials, +): + """ + Regression for #27516: when defaultconfig has multiple providers and the + request model has no '/' prefix, the deployment's custom_llm_provider must + drive provider matching instead of falling through to dict insertion order. + """ + litellm.credential_list.append( + CredentialItem( + credential_name="bedrock-team-1", + credential_info={}, + credential_values={"api_key": "ABSK-bedrock-key-for-team-1"}, + ) + ) + litellm.credential_list.append( + CredentialItem( + credential_name="gemini-team-1", + credential_info={}, + credential_values={"api_key": "gemini-key-for-team-1"}, + ) + ) + + data = {"model": "claude-sonnet-4.6"} + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + team_metadata={ + "model_config": { + "defaultconfig": { + # gemini comes first in insertion order — the bug picked it. + "gemini": {"litellm_credentials": "gemini-team-1"}, + "bedrock": {"litellm_credentials": "bedrock-team-1"}, + } + } + }, + ) + + router = MagicMock() + deployment = MagicMock() + deployment.litellm_params.model = "us.anthropic.claude-sonnet-4-6" + deployment.litellm_params.custom_llm_provider = "bedrock" + router.get_deployment_by_model_group_name.return_value = deployment + + _apply_credential_overrides_from_model_config( + data=data, + user_api_key_dict=user_api_key_dict, + llm_router=router, + ) + assert data["api_key"] == "ABSK-bedrock-key-for-team-1" + + +def test_apply_overrides_no_router_keeps_legacy_behaviour(setup_test_credentials): + """ + Without a router, the function still works for the single-provider case + (the historical behaviour). Multi-provider configs with no '/' prefix + keep the legacy first-entry behaviour because there is no way to + disambiguate — this preserves backwards compatibility. + """ + data = {"model": "gpt-4"} + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + team_metadata={ + "model_config": { + "defaultconfig": { + "azure": {"litellm_credentials": "hotel-azure-eastus"} + } + } + }, + ) + _apply_credential_overrides_from_model_config( + data=data, user_api_key_dict=user_api_key_dict, llm_router=None + ) + assert data["api_base"] == "https://hotel-eastus.openai.azure.com/" + assert data["api_key"] == "key-hotel-eastus" + + +def test_apply_overrides_provider_prefix_in_model_skips_router_lookup( + setup_test_credentials, +): + """ + When the request model already has a 'provider/...' prefix, the router + lookup must be skipped — the explicit prefix is authoritative. + """ + data = {"model": "azure/gpt-4"} + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + team_metadata={ + "model_config": { + "defaultconfig": { + "azure": {"litellm_credentials": "hotel-azure-eastus"}, + "bedrock": {"litellm_credentials": "hotel-rec-azure"}, + } + } + }, + ) + + router = MagicMock() + _apply_credential_overrides_from_model_config( + data=data, user_api_key_dict=user_api_key_dict, llm_router=router + ) + assert data["api_base"] == "https://hotel-eastus.openai.azure.com/" + assert data["api_key"] == "key-hotel-eastus" + router.get_deployment_by_model_group_name.assert_not_called() From f8b078b749f342a9e803a68af6d6e4b16e6c2ece Mon Sep 17 00:00:00 2001 From: Tai An Date: Sat, 9 May 2026 13:26:37 -0700 Subject: [PATCH 44/85] fix(bedrock/messages): preserve compact_20260112 context_management on /v1/messages (#27534) Squash-merged by litellm-agent from Anai-Guo's PR. --- .../anthropic_claude3_transformation.py | 51 +++++++++++- litellm/types/llms/bedrock.py | 7 ++ .../test_anthropic_claude3_transformation.py | 79 ++++++++++++++++++- 3 files changed, 132 insertions(+), 5 deletions(-) diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index aae2bc5e289..151e0e404a0 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -408,6 +408,47 @@ class AmazonAnthropicClaudeMessagesConfig( if self._supports_tool_search_on_bedrock(model): beta_set.add("tool-search-tool-2025-10-19") + @staticmethod + def _filter_context_management_for_bedrock_invoke( + anthropic_messages_request: Dict, + beta_set: set, + ) -> None: + """ + Bedrock InvokeModel accepts ``context_management`` only when it carries + ``compact_20260112`` edits paired with the ``compact-2026-01-12`` + anthropic-beta header. Other edit types (notably ``clear_thinking_20251015``, + which Claude Code sends on every request) are LiteLLM-internal and would + cause Bedrock to 400 with ``"context_management: Extra inputs are not + permitted"``. + + Filter the edits list to the supported subset, add the beta header when + compact edits remain, and drop ``context_management`` entirely when no + supported edits are left so the safety-net allowlist can pass it through. + + Ref: https://github.com/BerriAI/litellm/issues/27532 + """ + cm = anthropic_messages_request.get("context_management") + if not isinstance(cm, dict): + return + edits = cm.get("edits") + if not isinstance(edits, list): + anthropic_messages_request.pop("context_management", None) + return + + compact_edits = [ + e + for e in edits + if isinstance(e, dict) and e.get("type") == "compact_20260112" + ] + if compact_edits: + beta_set.add("compact-2026-01-12") + anthropic_messages_request["context_management"] = { + **cm, + "edits": compact_edits, + } + else: + anthropic_messages_request.pop("context_management", None) + def _convert_output_format_to_inline_schema( self, output_format: Dict, @@ -551,6 +592,11 @@ class AmazonAnthropicClaudeMessagesConfig( if injected_thinking_for_clear_thinking: beta_set.add("interleaved-thinking-2025-05-14") + self._filter_context_management_for_bedrock_invoke( + anthropic_messages_request=anthropic_messages_request, + beta_set=beta_set, + ) + self._get_tool_search_beta_header_for_bedrock( model=model, tool_search_used=tool_search_used, @@ -597,8 +643,9 @@ class AmazonAnthropicClaudeMessagesConfig( anthropic_messages_request.pop("output_config", None) # 7. Final safety net: filter top-level fields to the Bedrock Invoke allowlist. - # Catches Anthropic-only extensions (context_management, output_config, speed, - # mcp_servers, ...) and any future additions Claude Code may start sending. + # Catches Anthropic-only extensions (output_config, speed, mcp_servers, ...) + # and any future additions Claude Code may start sending. ``context_management`` + # has already been pre-filtered to its Bedrock-supported subset above. allowed = self.BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS stripped = sorted(k for k in anthropic_messages_request if k not in allowed) if stripped: diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index 64827db13f6..5db2a45054a 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -1042,3 +1042,10 @@ class BedrockInvokeAnthropicMessagesRequest(TypedDict, total=False): thinking: dict metadata: dict output_config: dict + + # `context_management` is allowed for Bedrock InvokeModel only when it + # carries `compact_20260112` edits paired with the `compact-2026-01-12` + # anthropic-beta header. The Invoke transformation filters edits to the + # supported subset and strips the field entirely when nothing remains, so + # other edit types (e.g. `clear_thinking_20251015`) never reach Bedrock. + context_management: dict diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index c98f7840343..9ecdad1fcff 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -867,10 +867,12 @@ def test_bedrock_messages_explicit_output_config_wins_over_reasoning_effort(): def test_bedrock_messages_strips_context_management(): """ Ensure context_management is stripped from the request before sending to - Bedrock Invoke, which doesn't support this Anthropic-specific parameter. + Bedrock Invoke when it carries only LiteLLM-internal edits (e.g. + clear_thinking_20251015, which is consumed via thinking injection). - Claude Code sends context_management on every request; leaving it in the body - causes a 400 "context_management: Extra inputs are not permitted" from Bedrock. + Claude Code sends context_management on every request; leaving such edits + in the body causes a 400 "context_management: Extra inputs are not + permitted" from Bedrock. """ from litellm.types.router import GenericLiteLLMParams @@ -897,6 +899,77 @@ def test_bedrock_messages_strips_context_management(): assert result.get("max_tokens") == 4096 +def test_bedrock_messages_preserves_compact_context_management_and_adds_beta(): + """ + Bedrock InvokeModel supports compaction when paired with the + ``compact-2026-01-12`` anthropic-beta header, even though the Converse API + does not. The transformation should: + 1. Keep ``context_management`` with compact_20260112 edits in the body + (Bedrock rejects unknown top-level fields, but accepts this one with + the right beta). + 2. Auto-inject ``compact-2026-01-12`` into ``anthropic_beta``. + + Ref: https://github.com/BerriAI/litellm/issues/27532 + """ + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [{"role": "user", "content": [{"type": "text", "text": "Hi"}]}] + optional_params = { + "max_tokens": 4096, + "context_management": { + "edits": [{"type": "compact_20260112"}] + }, + } + + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-sonnet-4-6-20250929-v1:0", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result.get("context_management") == { + "edits": [{"type": "compact_20260112"}] + } + assert "compact-2026-01-12" in result.get("anthropic_beta", []) + assert result["max_tokens"] == 4096 + + +def test_bedrock_messages_filters_unsupported_context_management_edits(): + """ + Mixed edit lists must drop the LiteLLM-internal ``clear_thinking_20251015`` + entries while keeping ``compact_20260112`` and adding the compact beta. + """ + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [{"role": "user", "content": [{"type": "text", "text": "Hi"}]}] + optional_params = { + "max_tokens": 4096, + "context_management": { + "edits": [ + {"type": "clear_thinking_20251015", "keep": "all"}, + {"type": "compact_20260112"}, + ] + }, + } + + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-sonnet-4-6-20250929-v1:0", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result.get("context_management") == { + "edits": [{"type": "compact_20260112"}] + } + assert "compact-2026-01-12" in result.get("anthropic_beta", []) + + def test_bedrock_messages_allowlist_filters_anthropic_only_fields(): """ Bedrock Invoke rejects any top-level body field it doesn't recognize with From d1d240086239c9d639758bcba81e7514825e52ad Mon Sep 17 00:00:00 2001 From: Kris Xia Date: Sun, 10 May 2026 04:30:18 +0800 Subject: [PATCH 45/85] fix(router): register model info under responses/-stripped variant (#27531) Squash-merged by litellm-agent from krisxia0506's PR. --- litellm/router.py | 44 ++++++++++---- .../test_router_model_cost_isolation.py | 57 +++++++++++++++++++ 2 files changed, 91 insertions(+), 10 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 7512ee387dc..37295f1a7d2 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7076,11 +7076,11 @@ class Router: _shared_model_info = { k: v for k, v in _model_info.items() if k not in _custom_pricing_fields } - litellm.register_model( - model_cost={ - _model_name: _shared_model_info, - } - ) + _backend_alias_cost = {_model_name: _shared_model_info} + if "responses/" in _model_name: + _stripped_model_name = _model_name.replace("responses/", "") + _backend_alias_cost[_stripped_model_name] = _shared_model_info + litellm.register_model(model_cost=_backend_alias_cost) ## Check if LLM Deployment is allowed for this deployment if ( @@ -7752,6 +7752,12 @@ class Router: # initialize client self._add_deployment(deployment=deployment) + _model_info_dict: dict = deployment.model_info.model_dump(exclude_none=True) + for field in CustomPricingLiteLLMParams.model_fields.keys(): + field_value = deployment.litellm_params.get(field) + if field_value is not None: + _model_info_dict[field] = field_value + # Register custom pricing in litellm.model_cost. # Mirrors _create_deployment() logic to ensure dynamically-added deployments # (e.g., loaded from DB) also have their custom pricing registered. @@ -7759,13 +7765,31 @@ class Router: # zero-cost models, causing budget checks to block free models. _model_id = deployment.model_info.id if _model_id is not None: - _model_info_dict: dict = deployment.model_info.model_dump(exclude_none=True) - for field in CustomPricingLiteLLMParams.model_fields.keys(): - field_value = deployment.litellm_params.get(field) - if field_value is not None: - _model_info_dict[field] = field_value litellm.register_model(model_cost={_model_id: _model_info_dict}) + ## REGISTER MODEL INFO IN LITELLM MODEL COST MAP + ## OLD MODEL REGISTRATION ## Kept to prevent breaking changes + _model_name = deployment.litellm_params.model + if deployment.litellm_params.custom_llm_provider is not None: + _model_name = ( + deployment.litellm_params.custom_llm_provider + "/" + _model_name + ) + + # For the shared backend key, strip custom pricing fields so that + # one deployment's pricing overrides don't pollute another + # deployment sharing the same backend model name. + # Each deployment's full pricing is already stored under its + # unique model_id above (when present). + _custom_pricing_fields = CustomPricingLiteLLMParams.model_fields.keys() + _shared_model_info = { + k: v for k, v in _model_info_dict.items() if k not in _custom_pricing_fields + } + _backend_alias_cost = {_model_name: _shared_model_info} + if "responses/" in _model_name: + _stripped_model_name = _model_name.replace("responses/", "") + _backend_alias_cost[_stripped_model_name] = _shared_model_info + litellm.register_model(model_cost=_backend_alias_cost) + # add to model names self._add_model_to_list_and_index_map( model=_deployment, model_id=deployment.model_info.id diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 7a9d5acaa27..c3f93078557 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -18,6 +18,7 @@ sys.path.insert( import litellm from litellm import Router +from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo def test_should_not_pollute_shared_key_with_zero_cost_pricing(): @@ -266,3 +267,59 @@ def test_should_preserve_builtin_pricing_regardless_of_deployment_order(): f"Order should not matter. Expected {builtin_output_cost}, " f"got {info_std_2['output_cost_per_token']}" ) + + +def test_responses_prefix_stripped_alias_registered_for_model_list(): + """ + Register ``litellm.model_cost`` under the backend key with ``responses/`` and + under the stripped key (``responses_api_bridge_check`` removes that segment). + """ + uid = "responses-strip-alias-test-a1b2c3d4" + Router( + model_list=[ + { + "model_name": "azure-responses-strip-test", + "litellm_params": { + "model": "responses/gpt-strip-test-a1b2c3d4", + "custom_llm_provider": "azure", + "api_key": "fake-key-strip", + }, + "model_info": { + "id": uid, + "supports_native_streaming": True, + }, + } + ], + ) + assert "azure/responses/gpt-strip-test-a1b2c3d4" in litellm.model_cost + assert "azure/gpt-strip-test-a1b2c3d4" in litellm.model_cost + assert ( + litellm.model_cost["azure/gpt-strip-test-a1b2c3d4"].get( + "supports_native_streaming" + ) + is True + ) + + +def test_responses_prefix_stripped_alias_registered_for_add_deployment(): + """Dynamic ``add_deployment`` must mirror ``_create_deployment`` registration.""" + uid = "add-dep-responses-strip-e5f6a7b8" + router = Router(model_list=[]) + deployment = Deployment( + model_name="dyn-responses-strip", + litellm_params=LiteLLM_Params( + model="responses/gpt-add-strip-e5f6a7b8", + custom_llm_provider="azure", + api_key="fake-key-add", + ), + model_info=ModelInfo(id=uid, supports_native_streaming=True), + ) + router.add_deployment(deployment=deployment) + assert "azure/responses/gpt-add-strip-e5f6a7b8" in litellm.model_cost + assert "azure/gpt-add-strip-e5f6a7b8" in litellm.model_cost + assert ( + litellm.model_cost["azure/gpt-add-strip-e5f6a7b8"].get( + "supports_native_streaming" + ) + is True + ) From b799f94080bcc0b17cd5644c8fa7a78aaf32bd24 Mon Sep 17 00:00:00 2001 From: Rick <26716961+Bytechoreographer@users.noreply.github.com> Date: Sun, 10 May 2026 04:30:36 +0800 Subject: [PATCH 46/85] fix(ui): remove blank leading entry from access group model dropdown (#27521) Squash-merged by litellm-agent from Bytechoreographer's PR. --- .../ModelSelect/ModelSelect.test.tsx | 15 +++++ .../components/ModelSelect/ModelSelect.tsx | 62 ++++++++++--------- 2 files changed, 47 insertions(+), 30 deletions(-) diff --git a/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.test.tsx b/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.test.tsx index 6da2f82a2f1..a57b7e2095a 100644 --- a/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.test.tsx +++ b/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.test.tsx @@ -535,6 +535,21 @@ describe("ModelSelect", () => { }); }); + it("should not render an empty optgroup when includeSpecialOptions is omitted", async () => { + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByTestId("model-select")).toBeInTheDocument(); + }); + + const optgroups = document.querySelectorAll("optgroup"); + // Wildcard Options + Models — no blank leading group + expect(optgroups.length).toBe(2); + optgroups.forEach((g) => { + expect(g.getAttribute("label")).toBeTruthy(); + }); + }); + it("should render maxTagPlaceholder when many items are selected", async () => { // Create many models to trigger maxTagCount responsive behavior const manyModels: ProxyModel[] = Array.from({ length: 20 }, (_, i) => ({ diff --git a/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx b/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx index 74b2619f7f3..800fa86b165 100644 --- a/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx +++ b/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx @@ -141,36 +141,38 @@ export const ModelSelect = (props: ModelSelectProps) => { onChange={handleChange} style={style} options={[ - includeSpecialOptions - ? { - label: Special Options, - title: "Special Options", - options: [ - ...(shouldShowAllProxyModels - ? [ - { - label: All Proxy Models, - value: MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE.value, - disabled: - value.length > 0 && - value.some( - (v) => isSpecialOption(v) && v !== MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE.value, - ), - key: MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE.value, - }, - ] - : []), - { - label: No Default Models, - value: MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE.value, - disabled: - value.length > 0 && - value.some((v) => isSpecialOption(v) && v !== MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE.value), - key: MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE.value, - }, - ], - } - : [], + ...(includeSpecialOptions + ? [ + { + label: Special Options, + title: "Special Options", + options: [ + ...(shouldShowAllProxyModels + ? [ + { + label: All Proxy Models, + value: MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE.value, + disabled: + value.length > 0 && + value.some( + (v) => isSpecialOption(v) && v !== MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE.value, + ), + key: MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE.value, + }, + ] + : []), + { + label: No Default Models, + value: MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE.value, + disabled: + value.length > 0 && + value.some((v) => isSpecialOption(v) && v !== MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE.value), + key: MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE.value, + }, + ], + }, + ] + : []), ...(wildcard.length > 0 ? [ { From 80445299b884ff0a20f5866da680d61d4340e4c8 Mon Sep 17 00:00:00 2001 From: Tai An Date: Sat, 9 May 2026 13:32:31 -0700 Subject: [PATCH 47/85] fix(proxy): coerce non-str x-litellm-* header values to avoid httpx TypeError (#27458) (#27504) Squash-merged by litellm-agent from Anai-Guo's PR. --- litellm/proxy/litellm_pre_call_utils.py | 12 +++++++++++- tests/proxy_unit_tests/test_proxy_utils.py | 13 +++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 9d7d6f476ac..b97e7c5e693 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1,5 +1,6 @@ import asyncio import copy +import json import re import time from collections import OrderedDict @@ -794,8 +795,17 @@ class LiteLLMProxyRequestSetup: ) ) for k, v in litellm_logging_metadata_headers.items(): - if v is not None: + if v is None: + continue + # httpx requires header values to be str or bytes; coerce numbers/bools + # to str and JSON-encode dict/list (e.g. user_api_key_spend is float, + # user_api_key_auth_metadata is dict). See #27458. + if isinstance(v, (dict, list)): + returned_headers["x-litellm-{}".format(k)] = json.dumps(v) + elif isinstance(v, (str, bytes)): returned_headers["x-litellm-{}".format(k)] = v + else: + returned_headers["x-litellm-{}".format(k)] = str(v) return returned_headers diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 70232f25c37..68aff36038b 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -587,12 +587,21 @@ def test_foward_litellm_user_info_to_backend_llm_call(): user_api_key_dict=user_api_key_dict, ) + # All header values must be str/bytes so httpx won't reject them when the + # downstream client builds the request (regression: #27458). + for k, v in data.items(): + assert isinstance(v, (str, bytes)), ( + f"header {k!r} has non-str value {v!r} ({type(v).__name__}); " + "httpx will raise 'Header value must be str or bytes' when the LLM " + "request is built." + ) + expected_data = { "x-litellm-user_api_key_user_id": "test_user_id", "x-litellm-user_api_key_org_id": "test_org_id", "x-litellm-user_api_key_hash": "test_api_key", - "x-litellm-user_api_key_spend": 0.0, - "x-litellm-user_api_key_auth_metadata": {}, + "x-litellm-user_api_key_spend": "0.0", + "x-litellm-user_api_key_auth_metadata": "{}", } assert json.dumps(data, sort_keys=True) == json.dumps(expected_data, sort_keys=True) From 8686001b3b56463a8564a640a44eff50c595efdf Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 9 May 2026 13:50:22 -0700 Subject: [PATCH 48/85] build(packaging): raise jinja2 floor to 3.1.6 Our `uv.lock` already resolves jinja2 to 3.1.6, so Docker / CI installs get that version. The `pyproject.toml` floor was lagging at 3.1.0, which means downstream consumers using `--resolution=lowest-direct` or older constraint files can land on 3.1.0-3.1.5 instead of the version we actually test against. Aligns the declared floor with the resolved version so external installers see the same baseline our test matrix exercises. `uv lock` diff is metadata-only (no resolved-version drift). --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d194d467913..5cd83148d37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "importlib-metadata>=8.0.0,<9.0", "tokenizers>=0.21.0,<1.0", "click>=8.0.0,<9.0", - "jinja2>=3.1.0,<4.0", + "jinja2>=3.1.6,<4.0", "aiohttp>=3.10,<4.0", "pydantic>=2.10.0,<3.0.0", "jsonschema>=4.0.0,<5.0", diff --git a/uv.lock b/uv.lock index f8d78fe8794..ab9aba1e38f 100644 --- a/uv.lock +++ b/uv.lock @@ -3405,7 +3405,7 @@ requires-dist = [ { name = "gunicorn", marker = "extra == 'proxy'", specifier = "==23.0.0" }, { name = "httpx", specifier = ">=0.28.0,<1.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, - { name = "jinja2", specifier = ">=3.1.0,<4.0" }, + { name = "jinja2", specifier = ">=3.1.6,<4.0" }, { name = "jsonschema", specifier = ">=4.0.0,<5.0" }, { name = "langfuse", marker = "extra == 'proxy-runtime'", specifier = "==2.59.7" }, { name = "litellm-enterprise", marker = "extra == 'proxy'", editable = "enterprise" }, From fa5eae8bc9d07d8e6293e97ed1a4e6120fe83cc5 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 9 May 2026 13:51:34 -0700 Subject: [PATCH 49/85] chore: remove legacy deployment artifacts and litellm-js packages (#27541) - Remove litellm-js/proxy and litellm-js/spend-logs TypeScript packages that provided Cloudflare Worker proxy and Node.js spend logging services, as these are no longer maintained - Remove deprecated Docker variants (Dockerfile.alpine, Dockerfile.dev, Dockerfile.custom_ui, Dockerfile.health_check, Dockerfile.ghcr_base) that have been superseded by the primary Dockerfile - Remove legacy Kubernetes manifests (kub.yaml, service.yaml) from deploy/kubernetes in favor of the Helm chart - Remove stale index.yaml Helm chart index pinned to an old version (v1.43.18) - Remove dev_config.yaml development configuration file that contained hardcoded credentials and example endpoints - Clean up ~3,500 lines of unused code and configuration to reduce repository maintenance burden Co-authored-by: Yassin Kortam --- AGENTS.md | 21 +- CLAUDE.md | 2 +- deploy/Dockerfile.ghcr_base | 18 - deploy/kubernetes/kub.yaml | 56 - deploy/kubernetes/service.yaml | 12 - dev_config.yaml | 13 - docker/Dockerfile.alpine | 68 - docker/Dockerfile.custom_ui | 86 - docker/Dockerfile.dev | 121 -- docker/Dockerfile.health_check | 30 - index.yaml | 108 -- litellm-js/proxy/.npmrc | 5 - litellm-js/proxy/README.md | 8 - litellm-js/proxy/package-lock.json | 2054 ----------------------- litellm-js/proxy/package.json | 14 - litellm-js/proxy/src/index.ts | 59 - litellm-js/proxy/tsconfig.json | 17 - litellm-js/proxy/wrangler.toml | 18 - litellm-js/spend-logs/.npmrc | 5 - litellm-js/spend-logs/Dockerfile | 26 - litellm-js/spend-logs/README.md | 8 - litellm-js/spend-logs/package-lock.json | 597 ------- litellm-js/spend-logs/package.json | 13 - litellm-js/spend-logs/schema.prisma | 29 - litellm-js/spend-logs/src/_types.ts | 32 - litellm-js/spend-logs/src/index.ts | 84 - litellm-js/spend-logs/tsconfig.json | 13 - 27 files changed, 20 insertions(+), 3497 deletions(-) delete mode 100644 deploy/Dockerfile.ghcr_base delete mode 100644 deploy/kubernetes/kub.yaml delete mode 100644 deploy/kubernetes/service.yaml delete mode 100644 dev_config.yaml delete mode 100644 docker/Dockerfile.alpine delete mode 100644 docker/Dockerfile.custom_ui delete mode 100644 docker/Dockerfile.dev delete mode 100644 docker/Dockerfile.health_check delete mode 100644 index.yaml delete mode 100644 litellm-js/proxy/.npmrc delete mode 100644 litellm-js/proxy/README.md delete mode 100644 litellm-js/proxy/package-lock.json delete mode 100644 litellm-js/proxy/package.json delete mode 100644 litellm-js/proxy/src/index.ts delete mode 100644 litellm-js/proxy/tsconfig.json delete mode 100644 litellm-js/proxy/wrangler.toml delete mode 100644 litellm-js/spend-logs/.npmrc delete mode 100644 litellm-js/spend-logs/Dockerfile delete mode 100644 litellm-js/spend-logs/README.md delete mode 100644 litellm-js/spend-logs/package-lock.json delete mode 100644 litellm-js/spend-logs/package.json delete mode 100644 litellm-js/spend-logs/schema.prisma delete mode 100644 litellm-js/spend-logs/src/_types.ts delete mode 100644 litellm-js/spend-logs/src/index.ts delete mode 100644 litellm-js/spend-logs/tsconfig.json diff --git a/AGENTS.md b/AGENTS.md index 4bdbf26ae9d..e99bf79d783 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -241,10 +241,27 @@ When opening issues or pull requests, follow these templates: ### Running the proxy server -Start the proxy with a config file: +Create a minimal config file and start the proxy: + +```yaml +# config.yaml +model_list: + - model_name: fake-openai-endpoint + litellm_params: + model: openai/fake-model + api_key: fake-key + api_base: https://fake-api.example.com + +general_settings: + master_key: sk-1234 + +litellm_settings: + drop_params: True + telemetry: False +``` ```bash -uv run litellm --config dev_config.yaml --port 4000 +uv run litellm --config config.yaml --port 4000 ``` The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package. diff --git a/CLAUDE.md b/CLAUDE.md index 71e5af28ee7..938801df7c7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -146,7 +146,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components: - **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets. - **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields. - **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries. -- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`. +- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`. ### Setup Wizard (`litellm/setup_wizard.py`) - The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI). diff --git a/deploy/Dockerfile.ghcr_base b/deploy/Dockerfile.ghcr_base deleted file mode 100644 index 66e64e5b774..00000000000 --- a/deploy/Dockerfile.ghcr_base +++ /dev/null @@ -1,18 +0,0 @@ -# Use the provided base image -FROM ghcr.io/berriai/litellm:main-latest@sha256:7c311546c25e7bb6e8cafede9fcd3d0d622ac636b5c9418befaa32e85dfb0186 - -# Set the working directory to /app -WORKDIR /app - -# Copy the configuration file into the container at /app -COPY config.yaml . - -# Make sure your docker/entrypoint.sh is executable -# Convert Windows line endings to Unix -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh - -# Expose the necessary port -EXPOSE 4000/tcp - -# Override the CMD instruction with your desired command and arguments -CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug", "--run_gunicorn"] diff --git a/deploy/kubernetes/kub.yaml b/deploy/kubernetes/kub.yaml deleted file mode 100644 index d5ba500d8f0..00000000000 --- a/deploy/kubernetes/kub.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: litellm-deployment -spec: - replicas: 3 - selector: - matchLabels: - app: litellm - template: - metadata: - labels: - app: litellm - spec: - containers: - - name: litellm-container - image: ghcr.io/berriai/litellm:main-latest - imagePullPolicy: Always - env: - - name: AZURE_API_KEY - value: "d6f****" - - name: AZURE_API_BASE - value: "https://openai" - - name: LITELLM_MASTER_KEY - value: "sk-1234" - - name: DATABASE_URL - value: "postgresql://ishaan*********" - args: - - "--config" - - "/app/proxy_config.yaml" # Update the path to mount the config file - volumeMounts: # Define volume mount for proxy_config.yaml - - name: config-volume - mountPath: /app - readOnly: true - livenessProbe: - httpGet: - path: /health/liveliness - port: 4000 - initialDelaySeconds: 120 - periodSeconds: 15 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: /health/readiness - port: 4000 - initialDelaySeconds: 120 - periodSeconds: 15 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 10 - volumes: # Define volume to mount proxy_config.yaml - - name: config-volume - configMap: - name: litellm-config diff --git a/deploy/kubernetes/service.yaml b/deploy/kubernetes/service.yaml deleted file mode 100644 index 4751c837254..00000000000 --- a/deploy/kubernetes/service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: litellm-service -spec: - selector: - app: litellm - ports: - - protocol: TCP - port: 4000 - targetPort: 4000 - type: LoadBalancer \ No newline at end of file diff --git a/dev_config.yaml b/dev_config.yaml deleted file mode 100644 index 64e3c14703e..00000000000 --- a/dev_config.yaml +++ /dev/null @@ -1,13 +0,0 @@ -model_list: - - model_name: fake-openai-endpoint - litellm_params: - model: openai/fake-model - api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - -general_settings: - master_key: sk-1234 - -litellm_settings: - drop_params: True - telemetry: False diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine deleted file mode 100644 index 5de588cf4e4..00000000000 --- a/docker/Dockerfile.alpine +++ /dev/null @@ -1,68 +0,0 @@ -# Base image for building -ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856 - -# Runtime image -ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856 -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a - -FROM $UV_IMAGE AS uvbin - -FROM $LITELLM_BUILD_IMAGE AS builder - -WORKDIR /app - -COPY --from=uvbin /uv /usr/local/bin/uv -COPY --from=uvbin /uvx /usr/local/bin/uvx - -RUN apk add --no-cache gcc python3-dev musl-dev nodejs npm libsndfile - -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -# Copy dependency metadata first for layer caching -COPY pyproject.toml uv.lock ./ -COPY enterprise/pyproject.toml enterprise/ -COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ - -# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) -RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python3 - -# Copy full source tree -COPY . . - -# Install project and workspace packages (fast - deps already cached) -RUN uv sync --frozen --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python3 - -RUN prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -FROM $LITELLM_RUNTIME_IMAGE AS runtime - -RUN apk upgrade --no-cache && apk add --no-cache libsndfile nodejs npm - -WORKDIR /app -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -COPY --from=builder /app /app - -EXPOSE 4000/tcp - -ENTRYPOINT ["docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] diff --git a/docker/Dockerfile.custom_ui b/docker/Dockerfile.custom_ui deleted file mode 100644 index cc44893bf92..00000000000 --- a/docker/Dockerfile.custom_ui +++ /dev/null @@ -1,86 +0,0 @@ -# Use the provided base image -# NOTE: This is a dev/branch-specific tag. Update digest when the base image is rebuilt. -FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev - -# Set the working directory to /app -WORKDIR /app - -# Install Node.js and npm (adjust version as needed) -RUN apt-get update && apt-get upgrade -y \ - libxml2 \ - libexpat1 \ - openssl \ - libssl3 \ - git \ - libkrb5-3 \ - libglib2.0-0 \ - wget \ - libaom3 \ - libxslt1.1 \ - libgnutls30 \ - libc6 && \ - apt-get install -y --no-install-recommends nodejs npm && \ - npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ - GLOBAL="$(npm root -g)" && \ - find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ - done && \ - find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \ - sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \ - npm cache clean --force && \ - apt-get purge -y npm - -# Copy the UI source into the container -COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard - -# Set an environment variable for UI_BASE_PATH -# This can be overridden at build time -# set UI_BASE_PATH to "/ui" -ENV UI_BASE_PATH="/prod/ui" - -# Build the UI with the specified UI_BASE_PATH -WORKDIR /app/ui/litellm-dashboard -RUN npm ci -RUN UI_BASE_PATH=$UI_BASE_PATH npm run build - -# Create the destination directory -RUN mkdir -p /app/litellm/proxy/_experimental/out - -# Move the built files to the appropriate location -# Assuming the build output is in ./out directory -RUN rm -rf /app/litellm/proxy/_experimental/out/* && \ - mv ./out/* /app/litellm/proxy/_experimental/out/ - -# Switch back to the main app directory -WORKDIR /app - -# Make sure your docker/entrypoint.sh is executable -# Convert Windows line endings to Unix for entrypoint scripts -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh -RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -# Run as non-root user -RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser \ - && chown -R appuser:appuser /app -USER appuser - -# Expose the necessary port -EXPOSE 4000/tcp - -HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ - CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health')"] - -# Override the CMD instruction with your desired command and arguments -CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"] \ No newline at end of file diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev deleted file mode 100644 index ebc92a22d50..00000000000 --- a/docker/Dockerfile.dev +++ /dev/null @@ -1,121 +0,0 @@ -# Base image for building -ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d - -# Runtime image -ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a - -FROM $UV_IMAGE AS uvbin - -FROM $LITELLM_BUILD_IMAGE AS builder - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /usr/local/bin/uv -COPY --from=uvbin /uvx /usr/local/bin/uvx - -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc \ - g++ \ - python3-dev \ - libssl-dev \ - pkg-config \ - nodejs \ - npm \ - && rm -rf /var/lib/apt/lists/* - -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -# Copy dependency metadata first for layer caching -COPY pyproject.toml uv.lock ./ -COPY enterprise/pyproject.toml enterprise/ -COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ - -# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) -RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python - -# Copy full source tree -COPY . . - -# Build Admin UI before final sync -RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh - -# Install project and workspace packages (fast - deps already cached) -RUN uv sync --frozen --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python - -RUN prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -FROM $LITELLM_RUNTIME_IMAGE AS runtime - -USER root - -RUN apt-get update && apt-get upgrade -y \ - libxml2 \ - libexpat1 \ - openssl \ - libssl3 \ - git \ - libkrb5-3 \ - libglib2.0-0 \ - wget \ - libaom3 \ - libxslt1.1 \ - libgnutls30 \ - libc6 \ - && apt-get install -y --no-install-recommends \ - libssl3 \ - libatomic1 \ - nodejs \ - npm \ - && rm -rf /var/lib/apt/lists/* \ - && npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \ - && GLOBAL="$(npm root -g)" \ - && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ - done \ - && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \ - sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \ - && npm cache clean --force \ - && apt-get purge -y npm - -WORKDIR /app -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -COPY --from=builder /app /app - -EXPOSE 4000/tcp - -ENTRYPOINT ["docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] diff --git a/docker/Dockerfile.health_check b/docker/Dockerfile.health_check deleted file mode 100644 index a2e5cb9f71f..00000000000 --- a/docker/Dockerfile.health_check +++ /dev/null @@ -1,30 +0,0 @@ -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a -FROM $UV_IMAGE AS uvbin - -FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d - -WORKDIR /app - -# Copy the uv binary and the health check script. -COPY --from=uvbin /uv /usr/local/bin/uv -COPY pyproject.toml uv.lock /app/ -COPY scripts/health_check/health_check_client.py /app/health_check_client.py - -# Resolve and install the health-check dependencies from the project lockfile -# so the runtime image stays self-contained and reproducible. -RUN uv export --frozen --no-default-groups --only-group healthcheck --no-emit-project --no-hashes --output-file /tmp/health-check-requirements.txt \ - && uv pip install --system -r /tmp/health-check-requirements.txt \ - && rm /tmp/health-check-requirements.txt \ - && rm /app/pyproject.toml /app/uv.lock \ - && chmod +x /app/health_check_client.py - -# Run as non-root user -RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser -USER appuser - -# Health check -HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \ - CMD ["python", "/app/health_check_client.py", "--help"] - -# Set entrypoint -ENTRYPOINT ["python", "/app/health_check_client.py"] diff --git a/index.yaml b/index.yaml deleted file mode 100644 index 9b2461c36b5..00000000000 --- a/index.yaml +++ /dev/null @@ -1,108 +0,0 @@ -apiVersion: v1 -entries: - litellm-helm: - - apiVersion: v2 - appVersion: v1.43.18 - created: "2024-08-19T23:58:25.331689+08:00" - dependencies: - - condition: db.deployStandalone - name: postgresql - repository: oci://registry-1.docker.io/bitnamicharts - version: '>=13.3.0' - - condition: redis.enabled - name: redis - repository: oci://registry-1.docker.io/bitnamicharts - version: '>=18.0.0' - description: Call all LLM APIs using the OpenAI format - digest: 0411df3dc42868be8af3ad3e00cb252790e6bd7ad15f5b77f1ca5214573a8531 - name: litellm-helm - type: application - urls: - - https://berriai.github.io/litellm/litellm-helm-0.2.3.tgz - version: 0.2.3 - postgresql: - - annotations: - category: Database - images: | - - name: os-shell - image: docker.io/bitnami/os-shell:12-debian-12-r16 - - name: postgres-exporter - image: docker.io/bitnami/postgres-exporter:0.15.0-debian-12-r14 - - name: postgresql - image: docker.io/bitnami/postgresql:16.2.0-debian-12-r6 - licenses: Apache-2.0 - apiVersion: v2 - appVersion: 16.2.0 - created: "2024-08-19T23:58:25.335716+08:00" - dependencies: - - name: common - repository: oci://registry-1.docker.io/bitnamicharts - tags: - - bitnami-common - version: 2.x.x - description: PostgreSQL (Postgres) is an open source object-relational database - known for reliability and data integrity. ACID-compliant, it supports foreign - keys, joins, views, triggers and stored procedures. - digest: 3c8125526b06833df32e2f626db34aeaedb29d38f03d15349db6604027d4a167 - home: https://bitnami.com - icon: https://bitnami.com/assets/stacks/postgresql/img/postgresql-stack-220x234.png - keywords: - - postgresql - - postgres - - database - - sql - - replication - - cluster - maintainers: - - name: VMware, Inc. - url: https://github.com/bitnami/charts - name: postgresql - sources: - - https://github.com/bitnami/charts/tree/main/bitnami/postgresql - urls: - - https://berriai.github.io/litellm/charts/postgresql-14.3.1.tgz - version: 14.3.1 - redis: - - annotations: - category: Database - images: | - - name: kubectl - image: docker.io/bitnami/kubectl:1.29.2-debian-12-r3 - - name: os-shell - image: docker.io/bitnami/os-shell:12-debian-12-r16 - - name: redis - image: docker.io/bitnami/redis:7.2.4-debian-12-r9 - - name: redis-exporter - image: docker.io/bitnami/redis-exporter:1.58.0-debian-12-r4 - - name: redis-sentinel - image: docker.io/bitnami/redis-sentinel:7.2.4-debian-12-r7 - licenses: Apache-2.0 - apiVersion: v2 - appVersion: 7.2.4 - created: "2024-08-19T23:58:25.339392+08:00" - dependencies: - - name: common - repository: oci://registry-1.docker.io/bitnamicharts - tags: - - bitnami-common - version: 2.x.x - description: Redis(R) is an open source, advanced key-value store. It is often - referred to as a data structure server since keys can contain strings, hashes, - lists, sets and sorted sets. - digest: b2fa1835f673a18002ca864c54fadac3c33789b26f6c5e58e2851b0b14a8f984 - home: https://bitnami.com - icon: https://bitnami.com/assets/stacks/redis/img/redis-stack-220x234.png - keywords: - - redis - - keyvalue - - database - maintainers: - - name: VMware, Inc. - url: https://github.com/bitnami/charts - name: redis - sources: - - https://github.com/bitnami/charts/tree/main/bitnami/redis - urls: - - https://berriai.github.io/litellm/charts/redis-18.19.1.tgz - version: 18.19.1 -generated: "2024-08-19T23:58:25.322532+08:00" diff --git a/litellm-js/proxy/.npmrc b/litellm-js/proxy/.npmrc deleted file mode 100644 index 7999681cc35..00000000000 --- a/litellm-js/proxy/.npmrc +++ /dev/null @@ -1,5 +0,0 @@ -# Supply-chain hardening -# Packages needing lifecycle scripts: npm rebuild -ignore-scripts=true -# Protects local npm install only — npm ci (used in CI) ignores this -min-release-age=3 diff --git a/litellm-js/proxy/README.md b/litellm-js/proxy/README.md deleted file mode 100644 index cc58e962d8f..00000000000 --- a/litellm-js/proxy/README.md +++ /dev/null @@ -1,8 +0,0 @@ -``` -npm install -npm run dev -``` - -``` -npm run deploy -``` diff --git a/litellm-js/proxy/package-lock.json b/litellm-js/proxy/package-lock.json deleted file mode 100644 index 0d09fa1a6c4..00000000000 --- a/litellm-js/proxy/package-lock.json +++ /dev/null @@ -1,2054 +0,0 @@ -{ - "name": "proxy", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "hono": "4.12.16", - "openai": "4.29.2" - }, - "devDependencies": { - "@cloudflare/workers-types": "4.20260501.1", - "wrangler": "4.87.0" - } - }, - "node_modules/@cloudflare/kv-asset-handler": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", - "integrity": "sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==", - "dev": true, - "license": "MIT OR Apache-2.0", - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@cloudflare/unenv-preset": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.16.1.tgz", - "integrity": "sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==", - "dev": true, - "license": "MIT OR Apache-2.0", - "peerDependencies": { - "unenv": "2.0.0-rc.24", - "workerd": ">1.20260305.0 <2.0.0-0" - }, - "peerDependenciesMeta": { - "workerd": { - "optional": true - } - } - }, - "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260430.1.tgz", - "integrity": "sha512-ADohZUHf7NBvPp2PdZig2Opxx+hDkk3ve7jrTne3JRx9kDSB73zc4LzcEeEN8LKkbAcqZmvfRJfpChSlusu0lA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260430.1.tgz", - "integrity": "sha512-/DoYC/1wHs+YRZzzqSQg1/EHB4hiv1yV5U8FnmapRRIzVaPtnt+ApeOXeMrIdKidgKOI8TqQzgBU8xbIM7Cl4Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260430.1.tgz", - "integrity": "sha512-koJhBWvEVZPKCVFtMLp2iMHlYr+lFCF47wGbnlKdHVlemV0zTxJEyHI8aLlrhPLhBmOmYLp46rXw09/qJkRIhQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260430.1.tgz", - "integrity": "sha512-hMdapNAzNQZDXGGkg4Slydc3fRJP5FUZLJVVcZCW/+imhhJro9Z1rv5n/wfR+txKoSWhTYR8eOp8Pyi2bzLzlw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260430.1.tgz", - "integrity": "sha512-jS3ffixjb5USOwz4frw4WzCz0HrjVxkgyU3WiYb06N7hBAfN6eOrveAJ4QRef0+suK4V1vQFoB1oKdRBsXe9Dw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workers-types": { - "version": "4.20260501.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260501.1.tgz", - "integrity": "sha512-B/VX2w3my/sCqxKyWOX7SxUpFC1uD8Gh7I2zbI1d3zA8p7Tx03AFsnuEx8lYLmcd8yONAA93YsAZb1wAaLK83w==", - "dev": true, - "license": "MIT OR Apache-2.0" - }, - "node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", - "integrity": "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.3.tgz", - "integrity": "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz", - "integrity": "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.3.tgz", - "integrity": "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz", - "integrity": "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz", - "integrity": "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz", - "integrity": "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz", - "integrity": "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz", - "integrity": "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz", - "integrity": "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz", - "integrity": "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz", - "integrity": "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz", - "integrity": "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz", - "integrity": "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz", - "integrity": "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz", - "integrity": "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.3.tgz", - "integrity": "sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz", - "integrity": "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz", - "integrity": "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz", - "integrity": "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz", - "integrity": "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz", - "integrity": "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz", - "integrity": "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz", - "integrity": "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz", - "integrity": "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz", - "integrity": "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/colour": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", - "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/sharp-darwin-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", - "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-darwin-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", - "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", - "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", - "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", - "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", - "cpu": [ - "arm" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", - "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", - "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", - "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", - "cpu": [ - "riscv64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", - "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", - "cpu": [ - "s390x" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", - "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", - "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", - "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-linux-arm": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", - "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", - "cpu": [ - "arm" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", - "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-ppc64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", - "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-riscv64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", - "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", - "cpu": [ - "riscv64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-s390x": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", - "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", - "cpu": [ - "s390x" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", - "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", - "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", - "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-wasm32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", - "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", - "optional": true, - "dependencies": { - "@emnapi/runtime": "^1.7.0" - }, - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", - "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-ia32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", - "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", - "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", - "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - }, - "node_modules/@poppinss/colors": { - "version": "4.1.6", - "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", - "integrity": "sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==", - "dev": true, - "license": "MIT", - "dependencies": { - "kleur": "^4.1.5" - } - }, - "node_modules/@poppinss/dumper": { - "version": "0.6.5", - "resolved": "https://registry.npmjs.org/@poppinss/dumper/-/dumper-0.6.5.tgz", - "integrity": "sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@poppinss/colors": "^4.1.5", - "@sindresorhus/is": "^7.0.2", - "supports-color": "^10.0.0" - } - }, - "node_modules/@poppinss/exception": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/@poppinss/exception/-/exception-1.2.3.tgz", - "integrity": "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@sindresorhus/is": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", - "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sindresorhus/is?sponsor=1" - } - }, - "node_modules/@speed-highlight/core": { - "version": "1.2.15", - "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.15.tgz", - "integrity": "sha512-BMq1K3DsElxDWawkX6eLg9+CKJrTVGCBAWVuHXVUV2u0s2711qiChLSId6ikYPfxhdYocLNt3wWwSvDiTvFabw==", - "dev": true, - "license": "CC0-1.0" - }, - "node_modules/@types/node": { - "version": "18.19.130", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", - "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", - "license": "MIT", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/@types/node-fetch": { - "version": "2.6.13", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", - "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.4" - } - }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "license": "MIT", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "license": "MIT", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, - "node_modules/base-64": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", - "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==" - }, - "node_modules/blake3-wasm": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", - "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", - "dev": true, - "license": "MIT" - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/charenc": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", - "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", - "license": "BSD-3-Clause", - "engines": { - "node": "*" - } - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/cookie": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", - "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/crypt": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", - "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", - "license": "BSD-3-Clause", - "engines": { - "node": "*" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, - "node_modules/digest-fetch": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz", - "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==", - "license": "ISC", - "dependencies": { - "base-64": "^0.1.0", - "md5": "^2.3.0" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/error-stack-parser-es": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", - "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/antfu" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/esbuild": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", - "integrity": "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.3", - "@esbuild/android-arm": "0.27.3", - "@esbuild/android-arm64": "0.27.3", - "@esbuild/android-x64": "0.27.3", - "@esbuild/darwin-arm64": "0.27.3", - "@esbuild/darwin-x64": "0.27.3", - "@esbuild/freebsd-arm64": "0.27.3", - "@esbuild/freebsd-x64": "0.27.3", - "@esbuild/linux-arm": "0.27.3", - "@esbuild/linux-arm64": "0.27.3", - "@esbuild/linux-ia32": "0.27.3", - "@esbuild/linux-loong64": "0.27.3", - "@esbuild/linux-mips64el": "0.27.3", - "@esbuild/linux-ppc64": "0.27.3", - "@esbuild/linux-riscv64": "0.27.3", - "@esbuild/linux-s390x": "0.27.3", - "@esbuild/linux-x64": "0.27.3", - "@esbuild/netbsd-arm64": "0.27.3", - "@esbuild/netbsd-x64": "0.27.3", - "@esbuild/openbsd-arm64": "0.27.3", - "@esbuild/openbsd-x64": "0.27.3", - "@esbuild/openharmony-arm64": "0.27.3", - "@esbuild/sunos-x64": "0.27.3", - "@esbuild/win32-arm64": "0.27.3", - "@esbuild/win32-ia32": "0.27.3", - "@esbuild/win32-x64": "0.27.3" - } - }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/form-data": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", - "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", - "license": "MIT" - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "license": "MIT", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, - "node_modules/formdata-node/node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", - "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/hono": { - "version": "4.12.16", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz", - "integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==", - "license": "MIT", - "engines": { - "node": ">=16.9.0" - } - }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.0.0" - } - }, - "node_modules/is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "license": "MIT" - }, - "node_modules/kleur": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", - "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/md5": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", - "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", - "license": "BSD-3-Clause", - "dependencies": { - "charenc": "0.0.2", - "crypt": "0.0.2", - "is-buffer": "~1.1.6" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/miniflare": { - "version": "4.20260430.0", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260430.0.tgz", - "integrity": "sha512-MWvMm3Siho9Yj7lbJZidLs8hbrRvIcOrif2mnsHQZdvoKfedpea+GaN8XJxbpRcq0B2WzNI1BB1ihdnqes3/ZA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@cspotcode/source-map-support": "0.8.1", - "sharp": "^0.34.5", - "undici": "7.24.8", - "workerd": "1.20260430.1", - "ws": "8.18.0", - "youch": "4.1.0-beta.10" - }, - "bin": { - "miniflare": "bootstrap.js" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/openai": { - "version": "4.29.2", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.29.2.tgz", - "integrity": "sha512-cPkT6zjEcE4qU5OW/SoDDuXEsdOLrXlAORhzmaguj5xZSPlgKvLhi27sFWhLKj07Y6WKNWxcwIbzm512FzTBNQ==", - "license": "Apache-2.0", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "digest-fetch": "^1.3.0", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7", - "web-streams-polyfill": "^3.2.1" - }, - "bin": { - "openai": "bin/cli" - } - }, - "node_modules/path-to-regexp": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", - "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/pathe": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", - "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", - "dev": true, - "license": "MIT" - }, - "node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/sharp": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", - "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", - "dev": true, - "hasInstallScript": true, - "license": "Apache-2.0", - "dependencies": { - "@img/colour": "^1.0.0", - "detect-libc": "^2.1.2", - "semver": "^7.7.3" - }, - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.34.5", - "@img/sharp-darwin-x64": "0.34.5", - "@img/sharp-libvips-darwin-arm64": "1.2.4", - "@img/sharp-libvips-darwin-x64": "1.2.4", - "@img/sharp-libvips-linux-arm": "1.2.4", - "@img/sharp-libvips-linux-arm64": "1.2.4", - "@img/sharp-libvips-linux-ppc64": "1.2.4", - "@img/sharp-libvips-linux-riscv64": "1.2.4", - "@img/sharp-libvips-linux-s390x": "1.2.4", - "@img/sharp-libvips-linux-x64": "1.2.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", - "@img/sharp-libvips-linuxmusl-x64": "1.2.4", - "@img/sharp-linux-arm": "0.34.5", - "@img/sharp-linux-arm64": "0.34.5", - "@img/sharp-linux-ppc64": "0.34.5", - "@img/sharp-linux-riscv64": "0.34.5", - "@img/sharp-linux-s390x": "0.34.5", - "@img/sharp-linux-x64": "0.34.5", - "@img/sharp-linuxmusl-arm64": "0.34.5", - "@img/sharp-linuxmusl-x64": "0.34.5", - "@img/sharp-wasm32": "0.34.5", - "@img/sharp-win32-arm64": "0.34.5", - "@img/sharp-win32-ia32": "0.34.5", - "@img/sharp-win32-x64": "0.34.5" - } - }, - "node_modules/supports-color": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", - "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD", - "optional": true - }, - "node_modules/undici": { - "version": "7.24.8", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.24.8.tgz", - "integrity": "sha512-6KQ/+QxK49Z/p3HO6E5ZCZWNnCasyZLa5ExaVYyvPxUwKtbCPMKELJOqh7EqOle0t9cH/7d2TaaTRRa6Nhs4YQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.18.1" - } - }, - "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "license": "MIT" - }, - "node_modules/unenv": { - "version": "2.0.0-rc.24", - "resolved": "https://registry.npmjs.org/unenv/-/unenv-2.0.0-rc.24.tgz", - "integrity": "sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "pathe": "^2.0.3" - } - }, - "node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/workerd": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260430.1.tgz", - "integrity": "sha512-KEgIWyiw3Jmn+DCd/L3ePo5fmiiYb/UcwKvDWPf/nLLOiwShDFzDSsegU5NY/JcwgvO/QsLHVi2FYrbkcXNY5Q==", - "dev": true, - "hasInstallScript": true, - "license": "Apache-2.0", - "bin": { - "workerd": "bin/workerd" - }, - "engines": { - "node": ">=16" - }, - "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20260430.1", - "@cloudflare/workerd-darwin-arm64": "1.20260430.1", - "@cloudflare/workerd-linux-64": "1.20260430.1", - "@cloudflare/workerd-linux-arm64": "1.20260430.1", - "@cloudflare/workerd-windows-64": "1.20260430.1" - } - }, - "node_modules/wrangler": { - "version": "4.87.0", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.87.0.tgz", - "integrity": "sha512-lfhfKwLfQlowwgV0xhlYgE9fU3n0I30d4ccGY/rTCEm/n42Mjvlr0Ng3ZPNqlsrsKBcDR531V7dsPkgELvrk/Q==", - "dev": true, - "license": "MIT OR Apache-2.0", - "dependencies": { - "@cloudflare/kv-asset-handler": "0.5.0", - "@cloudflare/unenv-preset": "2.16.1", - "blake3-wasm": "2.1.5", - "esbuild": "0.27.3", - "miniflare": "4.20260430.0", - "path-to-regexp": "6.3.0", - "unenv": "2.0.0-rc.24", - "workerd": "1.20260430.1" - }, - "bin": { - "wrangler": "bin/wrangler.js", - "wrangler2": "bin/wrangler.js" - }, - "engines": { - "node": ">=22.0.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - }, - "peerDependencies": { - "@cloudflare/workers-types": "^4.20260430.1" - }, - "peerDependenciesMeta": { - "@cloudflare/workers-types": { - "optional": true - } - } - }, - "node_modules/ws": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", - "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/youch": { - "version": "4.1.0-beta.10", - "resolved": "https://registry.npmjs.org/youch/-/youch-4.1.0-beta.10.tgz", - "integrity": "sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@poppinss/colors": "^4.1.5", - "@poppinss/dumper": "^0.6.4", - "@speed-highlight/core": "^1.2.7", - "cookie": "^1.0.2", - "youch-core": "^0.3.3" - } - }, - "node_modules/youch-core": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/youch-core/-/youch-core-0.3.3.tgz", - "integrity": "sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@poppinss/exception": "^1.2.2", - "error-stack-parser-es": "^1.0.5" - } - } - } -} diff --git a/litellm-js/proxy/package.json b/litellm-js/proxy/package.json deleted file mode 100644 index 9fd94cd882f..00000000000 --- a/litellm-js/proxy/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "scripts": { - "dev": "wrangler dev src/index.ts", - "deploy": "wrangler deploy --minify src/index.ts" - }, - "dependencies": { - "hono": "4.12.16", - "openai": "4.29.2" - }, - "devDependencies": { - "@cloudflare/workers-types": "4.20260501.1", - "wrangler": "4.87.0" - } -} diff --git a/litellm-js/proxy/src/index.ts b/litellm-js/proxy/src/index.ts deleted file mode 100644 index dc5dc9c689e..00000000000 --- a/litellm-js/proxy/src/index.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { Hono } from 'hono' -import { Context } from 'hono'; -import { bearerAuth } from 'hono/bearer-auth' -import OpenAI from "openai"; - -const openai = new OpenAI({ - apiKey: "sk-1234", - baseURL: "https://openai-endpoint.ishaanjaffer0324.workers.dev" -}); - -async function call_proxy() { - const completion = await openai.chat.completions.create({ - messages: [{ role: "system", content: "You are a helpful assistant." }], - model: "gpt-3.5-turbo", - }); - - return completion -} - -const app = new Hono() - -// Middleware for API Key Authentication -const apiKeyAuth = async (c: Context, next: Function) => { - const apiKey = c.req.header('Authorization'); - if (!apiKey || apiKey !== 'Bearer sk-1234') { - return c.text('Unauthorized', 401); - } - await next(); -}; - - -app.use('/*', apiKeyAuth) - - -app.get('/', (c) => { - return c.text('Hello Hono!') -}) - - - - -// Handler for chat completions -const chatCompletionHandler = async (c: Context) => { - // Assuming your logic for handling chat completion goes here - // For demonstration, just returning a simple JSON response - const response = await call_proxy() - return c.json(response); -}; - -// Register the above handler for different POST routes with the apiKeyAuth middleware -app.post('/v1/chat/completions', chatCompletionHandler); -app.post('/chat/completions', chatCompletionHandler); - -// Example showing how you might handle dynamic segments within the URL -// Here, using ':model*' to capture the rest of the path as a parameter 'model' -app.post('/openai/deployments/:model*/chat/completions', chatCompletionHandler); - - -export default app diff --git a/litellm-js/proxy/tsconfig.json b/litellm-js/proxy/tsconfig.json deleted file mode 100644 index 28fcfb58246..00000000000 --- a/litellm-js/proxy/tsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "Bundler", - "strict": true, - "lib": [ - "ESNext" - ], - "types": [ - "@cloudflare/workers-types" - ], - "jsx": "react-jsx", - "jsxImportSource": "hono/jsx", - "skipLibCheck": true - }, -} \ No newline at end of file diff --git a/litellm-js/proxy/wrangler.toml b/litellm-js/proxy/wrangler.toml deleted file mode 100644 index e7c323dff97..00000000000 --- a/litellm-js/proxy/wrangler.toml +++ /dev/null @@ -1,18 +0,0 @@ -name = "my-app" -compatibility_date = "2023-12-01" - -# [vars] -# MY_VAR = "my-variable" - -# [[kv_namespaces]] -# binding = "MY_KV_NAMESPACE" -# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - -# [[r2_buckets]] -# binding = "MY_BUCKET" -# bucket_name = "my-bucket" - -# [[d1_databases]] -# binding = "DB" -# database_name = "my-database" -# database_id = "" diff --git a/litellm-js/spend-logs/.npmrc b/litellm-js/spend-logs/.npmrc deleted file mode 100644 index 7999681cc35..00000000000 --- a/litellm-js/spend-logs/.npmrc +++ /dev/null @@ -1,5 +0,0 @@ -# Supply-chain hardening -# Packages needing lifecycle scripts: npm rebuild -ignore-scripts=true -# Protects local npm install only — npm ci (used in CI) ignores this -min-release-age=3 diff --git a/litellm-js/spend-logs/Dockerfile b/litellm-js/spend-logs/Dockerfile deleted file mode 100644 index 5040dc74bf6..00000000000 --- a/litellm-js/spend-logs/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Use the specific Node.js v20.11.0 image -FROM node:20.18.1-alpine3.20 - -# Set the working directory inside the container -WORKDIR /app - -# Copy package.json and package-lock.json to the working directory -COPY ./litellm-js/spend-logs/package*.json ./ - -# Install dependencies -RUN npm ci - -# Install Prisma globally -RUN npm install -g prisma - -# Copy the rest of the application code -COPY ./litellm-js/spend-logs . - -# Generate Prisma client -RUN npx prisma generate - -# Expose the port that the Node.js server will run on -EXPOSE 3000 - -# Command to run the Node.js app with npm run dev -CMD ["npm", "run", "dev"] diff --git a/litellm-js/spend-logs/README.md b/litellm-js/spend-logs/README.md deleted file mode 100644 index e12b31db70a..00000000000 --- a/litellm-js/spend-logs/README.md +++ /dev/null @@ -1,8 +0,0 @@ -``` -npm install -npm run dev -``` - -``` -open http://localhost:3000 -``` diff --git a/litellm-js/spend-logs/package-lock.json b/litellm-js/spend-logs/package-lock.json deleted file mode 100644 index e33079766c9..00000000000 --- a/litellm-js/spend-logs/package-lock.json +++ /dev/null @@ -1,597 +0,0 @@ -{ - "name": "spend-logs", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "@hono/node-server": "1.19.13", - "hono": "4.12.16" - }, - "devDependencies": { - "@types/node": "20.19.25", - "tsx": "4.20.6" - } - }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", - "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", - "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", - "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", - "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", - "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", - "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", - "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", - "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", - "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", - "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", - "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", - "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", - "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", - "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", - "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", - "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", - "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", - "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", - "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", - "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", - "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", - "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", - "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", - "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", - "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", - "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@hono/node-server": { - "version": "1.19.13", - "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.13.tgz", - "integrity": "sha512-TsQLe4i2gvoTtrHje625ngThGBySOgSK3Xo2XRYOdqGN1teR8+I7vchQC46uLJi8OF62YTYA3AhSpumtkhsaKQ==", - "license": "MIT", - "engines": { - "node": ">=18.14.1" - }, - "peerDependencies": { - "hono": "^4" - } - }, - "node_modules/@types/node": { - "version": "20.19.25", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.25.tgz", - "integrity": "sha512-ZsJzA5thDQMSQO788d7IocwwQbI8B5OPzmqNvpf3NY/+MHDAS759Wo0gd2WQeXYt5AAAQjzcrTVC6SKCuYgoCQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "node_modules/esbuild": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", - "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.12", - "@esbuild/android-arm": "0.25.12", - "@esbuild/android-arm64": "0.25.12", - "@esbuild/android-x64": "0.25.12", - "@esbuild/darwin-arm64": "0.25.12", - "@esbuild/darwin-x64": "0.25.12", - "@esbuild/freebsd-arm64": "0.25.12", - "@esbuild/freebsd-x64": "0.25.12", - "@esbuild/linux-arm": "0.25.12", - "@esbuild/linux-arm64": "0.25.12", - "@esbuild/linux-ia32": "0.25.12", - "@esbuild/linux-loong64": "0.25.12", - "@esbuild/linux-mips64el": "0.25.12", - "@esbuild/linux-ppc64": "0.25.12", - "@esbuild/linux-riscv64": "0.25.12", - "@esbuild/linux-s390x": "0.25.12", - "@esbuild/linux-x64": "0.25.12", - "@esbuild/netbsd-arm64": "0.25.12", - "@esbuild/netbsd-x64": "0.25.12", - "@esbuild/openbsd-arm64": "0.25.12", - "@esbuild/openbsd-x64": "0.25.12", - "@esbuild/openharmony-arm64": "0.25.12", - "@esbuild/sunos-x64": "0.25.12", - "@esbuild/win32-arm64": "0.25.12", - "@esbuild/win32-ia32": "0.25.12", - "@esbuild/win32-x64": "0.25.12" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/get-tsconfig": { - "version": "4.14.0", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz", - "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==", - "dev": true, - "license": "MIT", - "dependencies": { - "resolve-pkg-maps": "^1.0.0" - }, - "funding": { - "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" - } - }, - "node_modules/hono": { - "version": "4.12.16", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz", - "integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==", - "license": "MIT", - "engines": { - "node": ">=16.9.0" - } - }, - "node_modules/resolve-pkg-maps": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", - "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" - } - }, - "node_modules/tsx": { - "version": "4.20.6", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz", - "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==", - "dev": true, - "license": "MIT", - "dependencies": { - "esbuild": "~0.25.0", - "get-tsconfig": "^4.7.5" - }, - "bin": { - "tsx": "dist/cli.mjs" - }, - "engines": { - "node": ">=18.0.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - } - }, - "node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, - "license": "MIT" - } - } -} diff --git a/litellm-js/spend-logs/package.json b/litellm-js/spend-logs/package.json deleted file mode 100644 index 5a7a95c5de1..00000000000 --- a/litellm-js/spend-logs/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "scripts": { - "dev": "tsx watch src/index.ts" - }, - "dependencies": { - "@hono/node-server": "1.19.13", - "hono": "4.12.16" - }, - "devDependencies": { - "@types/node": "20.19.25", - "tsx": "4.20.6" - } -} diff --git a/litellm-js/spend-logs/schema.prisma b/litellm-js/spend-logs/schema.prisma deleted file mode 100644 index b0403f277aa..00000000000 --- a/litellm-js/spend-logs/schema.prisma +++ /dev/null @@ -1,29 +0,0 @@ -generator client { - provider = "prisma-client-js" -} - -datasource client { - provider = "postgresql" - url = env("DATABASE_URL") -} - -model LiteLLM_SpendLogs { - request_id String @id - call_type String - api_key String @default("") - spend Float @default(0.0) - total_tokens Int @default(0) - prompt_tokens Int @default(0) - completion_tokens Int @default(0) - startTime DateTime - endTime DateTime - model String @default("") - api_base String @default("") - user String @default("") - metadata Json @default("{}") - cache_hit String @default("") - cache_key String @default("") - request_tags Json @default("[]") - team_id String? - end_user String? -} \ No newline at end of file diff --git a/litellm-js/spend-logs/src/_types.ts b/litellm-js/spend-logs/src/_types.ts deleted file mode 100644 index 6a9b499171e..00000000000 --- a/litellm-js/spend-logs/src/_types.ts +++ /dev/null @@ -1,32 +0,0 @@ -export type LiteLLM_IncrementSpend = { - key_transactions: Array, // [{"key": spend},..] - user_transactions: Array, - team_transactions: Array, - spend_logs_transactions: Array -} - -export type LiteLLM_IncrementObject = { - key: string, - spend: number -} - -export type LiteLLM_SpendLogs = { - request_id: string; // @id means it's a unique identifier - call_type: string; - api_key: string; // @default("") means it defaults to an empty string if not provided - spend: number; // Float in Prisma corresponds to number in TypeScript - total_tokens: number; // Int in Prisma corresponds to number in TypeScript - prompt_tokens: number; - completion_tokens: number; - startTime: Date; // DateTime in Prisma corresponds to Date in TypeScript - endTime: Date; - model: string; // @default("") means it defaults to an empty string if not provided - api_base: string; - user: string; - metadata: any; // Json type in Prisma is represented by any in TypeScript; could also use a more specific type if the structure of JSON is known - cache_hit: string; - cache_key: string; - request_tags: any; // Similarly, this could be an array or a more specific type depending on the expected structure - team_id?: string | null; // ? indicates it's optional and can be undefined, but could also be null if not provided - end_user?: string | null; -}; \ No newline at end of file diff --git a/litellm-js/spend-logs/src/index.ts b/litellm-js/spend-logs/src/index.ts deleted file mode 100644 index 3581d95c830..00000000000 --- a/litellm-js/spend-logs/src/index.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { serve } from '@hono/node-server' -import { Hono } from 'hono' -import { PrismaClient } from '@prisma/client' -import {LiteLLM_SpendLogs, LiteLLM_IncrementSpend, LiteLLM_IncrementObject} from './_types' - -const app = new Hono() -const prisma = new PrismaClient() -// In-memory storage for logs -let spend_logs: LiteLLM_SpendLogs[] = []; -const key_logs: LiteLLM_IncrementObject[] = []; -const user_logs: LiteLLM_IncrementObject[] = []; -const transaction_logs: LiteLLM_IncrementObject[] = []; - - -app.get('/', (c) => { - return c.text('Hello Hono!') -}) - -const MIN_LOGS = 1; // Minimum number of logs needed to initiate a flush -const FLUSH_INTERVAL = 5000; // Time in ms to wait before trying to flush again -const BATCH_SIZE = 100; // Preferred size of each batch to write to the database -const MAX_LOGS_PER_INTERVAL = 1000; // Maximum number of logs to flush in a single interval - -const flushLogsToDb = async () => { - if (spend_logs.length >= MIN_LOGS) { - // Limit the logs to process in this interval to MAX_LOGS_PER_INTERVAL or less - const logsToProcess = spend_logs.slice(0, MAX_LOGS_PER_INTERVAL); - - for (let i = 0; i < logsToProcess.length; i += BATCH_SIZE) { - // Create subarray for current batch, ensuring it doesn't exceed the BATCH_SIZE - const batch = logsToProcess.slice(i, i + BATCH_SIZE); - - // Convert datetime strings to Date objects - const batchWithDates = batch.map(entry => ({ - ...entry, - startTime: new Date(entry.startTime), - endTime: new Date(entry.endTime), - // Repeat for any other DateTime fields you may have - })); - - await prisma.liteLLM_SpendLogs.createMany({ - data: batchWithDates, - }); - - console.log(`Flushed ${batch.length} logs to the DB.`); - } - - // Remove the processed logs from spend_logs - spend_logs = spend_logs.slice(logsToProcess.length); - - console.log(`${logsToProcess.length} logs processed. Remaining in queue: ${spend_logs.length}`); - } else { - // This will ensure it doesn't falsely claim "No logs to flush." when it's merely below the MIN_LOGS threshold. - if(spend_logs.length > 0) { - console.log(`Accumulating logs. Currently at ${spend_logs.length}, waiting for at least ${MIN_LOGS}.`); - } else { - console.log("No logs to flush."); - } - } -}; - -// Setup interval for attempting to flush the logs -setInterval(flushLogsToDb, FLUSH_INTERVAL); - -// Route to receive log messages -app.post('/spend/update', async (c) => { - const incomingLogs = await c.req.json(); - - spend_logs.push(...incomingLogs); - - console.log(`Received and stored ${incomingLogs.length} logs. Total logs in memory: ${spend_logs.length}`); - - return c.json({ message: `Successfully stored ${incomingLogs.length} logs` }); -}); - - - -const port = 3000 -console.log(`Server is running on port ${port}`) - -serve({ - fetch: app.fetch, - port -}) diff --git a/litellm-js/spend-logs/tsconfig.json b/litellm-js/spend-logs/tsconfig.json deleted file mode 100644 index 028c03b6a81..00000000000 --- a/litellm-js/spend-logs/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "Bundler", - "strict": true, - "types": [ - "node" - ], - "jsx": "react-jsx", - "jsxImportSource": "hono/jsx", - } -} \ No newline at end of file From d67dfca1e111cbb63f662cc31878d8e475113baa Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Sat, 9 May 2026 14:47:48 -0700 Subject: [PATCH 50/85] Fix proxy auth status code tests (#27555) * Fix proxy auth status code tests Co-authored-by: ishaan-berri * Update user model access status expectation Co-authored-by: ishaan-berri --------- Co-authored-by: oss-agent-shin <279349115+oss-agent-shin@users.noreply.github.com> Co-authored-by: ishaan-berri --- litellm/proxy/auth/auth_checks.py | 6 +-- litellm/proxy/auth/auth_exception_handler.py | 2 +- litellm/proxy/auth/user_api_key_auth.py | 6 +-- tests/otel_tests/test_e2e_budgeting.py | 4 +- tests/otel_tests/test_e2e_model_access.py | 4 +- .../proxy/auth/test_auth_checks.py | 49 +++++++++++++++++++ .../proxy/auth/test_auth_exception_handler.py | 1 + .../proxy/auth/test_user_api_key_auth.py | 23 +++++++++ tests/test_openai_endpoints.py | 2 +- tests/test_users.py | 4 +- 10 files changed, 87 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index f6f99eb62c8..0b30999aa21 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -2849,7 +2849,7 @@ def _can_object_call_model( object_type=object_type ), param="model", - code=status.HTTP_401_UNAUTHORIZED, + code=status.HTTP_403_FORBIDDEN, ) @@ -3082,7 +3082,7 @@ async def can_user_call_model( message=f"User not allowed to access model. No default model access, only team models allowed. Tried to access {model}", type=ProxyErrorTypes.key_model_access_denied, param="model", - code=status.HTTP_401_UNAUTHORIZED, + code=status.HTTP_403_FORBIDDEN, ) return _can_object_call_model( @@ -3625,7 +3625,7 @@ async def _check_team_member_model_access( message=f"Team member not allowed to access model. User={valid_token.user_id}, Team={team_object.team_id}, Model={model}. Allowed member models = {member_allowed_models}", type=ProxyErrorTypes.team_model_access_denied, param="model", - code=status.HTTP_401_UNAUTHORIZED, + code=status.HTTP_403_FORBIDDEN, ) diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index 5ded8136ef3..431db4254eb 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -123,7 +123,7 @@ class UserAPIKeyAuthExceptionHandler: message=e.message, type=ProxyErrorTypes.budget_exceeded, param=None, - code=400, + code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS), ) if isinstance(e, HTTPException): raise ProxyException( diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 9d3c06e641f..4778549befc 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1107,7 +1107,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 raise ProxyException( message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", type=ProxyErrorTypes.expired_key, - code=400, + code=status.HTTP_401_UNAUTHORIZED, param=abbreviate_api_key(api_key=api_key), ) valid_token = update_valid_token_with_end_user_params( @@ -1432,7 +1432,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 raise ProxyException( message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", type=ProxyErrorTypes.expired_key, - code=400, + code=status.HTTP_401_UNAUTHORIZED, param=abbreviate_api_key(api_key=api_key), ) @@ -2417,7 +2417,7 @@ async def _run_post_custom_auth_checks( raise ProxyException( message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", type=ProxyErrorTypes.expired_key, - code=400, + code=status.HTTP_401_UNAUTHORIZED, param=( abbreviate_api_key(api_key=valid_token.token) if valid_token.token diff --git a/tests/otel_tests/test_e2e_budgeting.py b/tests/otel_tests/test_e2e_budgeting.py index 62fc8732ebd..f61befac4fb 100644 --- a/tests/otel_tests/test_e2e_budgeting.py +++ b/tests/otel_tests/test_e2e_budgeting.py @@ -25,8 +25,8 @@ async def make_calls_until_budget_exceeded(session, key: str, call_function, **k # Check error structure and values that should be consistent assert ( - error_dict["code"] == "400" - ), f"Expected error code 400, got: {error_dict['code']}" + error_dict["code"] == "429" + ), f"Expected error code 429, got: {error_dict['code']}" assert ( error_dict["type"] == "budget_exceeded" ), f"Expected error type budget_exceeded, got: {error_dict['type']}" diff --git a/tests/otel_tests/test_e2e_model_access.py b/tests/otel_tests/test_e2e_model_access.py index 7ea75a9d61d..87d85a19603 100644 --- a/tests/otel_tests/test_e2e_model_access.py +++ b/tests/otel_tests/test_e2e_model_access.py @@ -99,7 +99,7 @@ async def test_model_access_patterns(key_models, test_model, expect_success): # Assert error structure and values assert _error_body["type"] == "key_model_access_denied" assert _error_body["param"] == "model" - assert _error_body["code"] == "401" + assert _error_body["code"] == "403" assert "key not allowed to access model" in _error_body["message"] @@ -297,7 +297,7 @@ def _validate_model_access_exception( # Assert error structure and values assert _error_body["type"] == expected_type assert _error_body["param"] == "model" - assert _error_body["code"] == "401" + assert _error_body["code"] == "403" if expected_type == "key_model_access_denied": assert "key not allowed to access model" in _error_body["message"] elif expected_type == "team_model_access_denied": diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 8a854bcd6a8..26f04a4abcb 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -12,6 +12,7 @@ from datetime import datetime, timedelta import httpx import pytest +from fastapi import status import litellm from litellm.proxy._types import ( @@ -31,6 +32,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.auth_checks import ( ExperimentalUIJWTToken, + _can_object_call_model, _can_object_call_vector_stores, _check_end_user_budget, _check_team_member_budget, @@ -206,6 +208,52 @@ def test_get_key_object_from_ui_hash_key_invalid(): assert key_object is None +@pytest.mark.parametrize( + "object_type,expected_error_type", + [ + ("key", ProxyErrorTypes.key_model_access_denied), + ("team", ProxyErrorTypes.team_model_access_denied), + ("user", ProxyErrorTypes.user_model_access_denied), + ("org", ProxyErrorTypes.org_model_access_denied), + ("project", ProxyErrorTypes.project_model_access_denied), + ], +) +def test_can_object_call_model_denials_return_forbidden( + object_type, expected_error_type +): + with pytest.raises(ProxyException) as exc_info: + _can_object_call_model( + model="restricted-model", + llm_router=None, + models=["allowed-model"], + object_type=object_type, + ) + + assert exc_info.value.type == expected_error_type + assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN + + +@pytest.mark.asyncio +async def test_can_user_call_model_no_default_models_returns_forbidden(): + from litellm.proxy._types import SpecialModelNames + from litellm.proxy.auth.auth_checks import can_user_call_model + + user_object = LiteLLM_UserTable( + user_id="test-user", + models=[SpecialModelNames.no_default_models.value], + ) + + with pytest.raises(ProxyException) as exc_info: + await can_user_call_model( + model="restricted-model", + llm_router=None, + user_object=user_object, + ) + + assert exc_info.value.type == ProxyErrorTypes.key_model_access_denied + assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN + + @pytest.mark.asyncio async def test_get_key_object_should_reconnect_once_on_db_connection_error(): mock_prisma_client = MagicMock() @@ -1144,6 +1192,7 @@ async def test_check_team_member_model_access_denied_model(): proxy_logging_obj=MagicMock(), ) assert exc_info.value.type == ProxyErrorTypes.team_model_access_denied + assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index 2d1586f0b17..4ccde85dae2 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -140,6 +140,7 @@ async def test_handle_authentication_error_budget_exceeded(): ) assert exc_info.value.type == ProxyErrorTypes.budget_exceeded + assert int(exc_info.value.code) == status.HTTP_429_TOO_MANY_REQUESTS @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 95b3d746c66..50c5f43b218 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -1,6 +1,7 @@ import json import os import sys +from datetime import datetime, timedelta from types import SimpleNamespace from unittest.mock import ANY, AsyncMock, MagicMock, patch @@ -9,6 +10,7 @@ sys.path.insert( ) # Adds the parent directory to the system path import pytest +from fastapi import status import litellm import litellm.proxy.proxy_server @@ -178,6 +180,26 @@ async def test_custom_auth_does_not_enforce_key_model_access_by_default(): mock_can_key.assert_not_awaited() +@pytest.mark.asyncio +async def test_post_custom_auth_expired_key_returns_unauthorized(): + expired_token = UserAPIKeyAuth( + token="test_token", + expires=datetime.now() - timedelta(minutes=1), + ) + + with pytest.raises(ProxyException) as exc_info: + await _run_post_custom_auth_checks( + valid_token=expired_token, + request=MagicMock(), + request_data={}, + route="/v1/chat/completions", + parent_otel_span=None, + ) + + assert exc_info.value.type == ProxyErrorTypes.expired_key + assert int(exc_info.value.code) == status.HTTP_401_UNAUTHORIZED + + @pytest.mark.asyncio async def test_custom_auth_honors_key_level_model_access_restriction_allowed_with_opt_in(): valid_token = UserAPIKeyAuth(token="test_token", models=["gpt-4o-mini"]) @@ -934,6 +956,7 @@ async def test_proxy_admin_expired_key_from_cache(): assert ( exc_info.value.type == ProxyErrorTypes.expired_key ), f"Expected expired_key error type, got {exc_info.value.type}" + assert int(exc_info.value.code) == status.HTTP_401_UNAUTHORIZED assert "Expired Key" in str( exc_info.value.message ), f"Exception message should mention 'Expired Key', got: {exc_info.value.message}" diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py index 024d05e1037..8a3f9361ba1 100644 --- a/tests/test_openai_endpoints.py +++ b/tests/test_openai_endpoints.py @@ -303,7 +303,7 @@ async def test_chat_completion(): api_key=key_gen["key"], api_version="2024-02-15-preview", ) - with pytest.raises(openai.AuthenticationError) as e: + with pytest.raises(openai.PermissionDeniedError) as e: response = await azure_client.chat.completions.create( model="gpt-4", messages=[{"role": "user", "content": "Hello!"}], diff --git a/tests/test_users.py b/tests/test_users.py index 05253a19aa5..57fbb0483e4 100644 --- a/tests/test_users.py +++ b/tests/test_users.py @@ -302,14 +302,14 @@ async def test_user_model_access(): model="good-model", ) - with pytest.raises(openai.AuthenticationError): + with pytest.raises(openai.PermissionDeniedError): await chat_completion( session=session, key=key, model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", ) - with pytest.raises(openai.AuthenticationError): + with pytest.raises(openai.PermissionDeniedError): await chat_completion( session=session, key=key, From b888177ea67a83511cdd67fbd55b820059ac7e81 Mon Sep 17 00:00:00 2001 From: Michael-RZ-Berri Date: Sat, 9 May 2026 15:33:36 -0700 Subject: [PATCH 51/85] fix: reset proxy budget when initial reset duration is null then updated (#27488) Co-authored-by: Michael Riad Zaky --- litellm/proxy/proxy_server.py | 74 ++++++++++++++----- tests/test_litellm/proxy/test_proxy_server.py | 61 +++++++++++++++ 2 files changed, 117 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 02f9c9bef2f..493519f2328 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -211,6 +211,7 @@ from litellm import Router from litellm._logging import verbose_proxy_logger, verbose_router_logger from litellm.caching.caching import DualCache, RedisCache from litellm.caching.redis_cluster_cache import RedisClusterCache +from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.constants import ( _REALTIME_BODY_CACHE_SIZE, @@ -6750,27 +6751,64 @@ class ProxyStartupEvent: "budget_duration not set on Proxy. budget_duration is required to use max_budget." ) - # add proxy budget to db in the user table asyncio.create_task( - generate_key_helper_fn( # type: ignore - request_type="user", - table_name="user", - user_id=litellm_proxy_budget_name, - duration=None, - models=[], - aliases={}, - config={}, - spend=0, - max_budget=litellm.max_budget, - budget_duration=litellm.budget_duration, - query_type="update_data", - update_key_values={ - "max_budget": litellm.max_budget, - "budget_duration": litellm.budget_duration, - }, - ) + cls._upsert_proxy_budget_with_reset_at_backfill(litellm_proxy_budget_name) ) + @classmethod + async def _upsert_proxy_budget_with_reset_at_backfill( + cls, litellm_proxy_budget_name: str + ) -> None: + """ + Upsert the proxy admin user row with the configured max_budget / + budget_duration, then backfill budget_reset_at if currently NULL. + + The backfill uses `WHERE budget_reset_at IS NULL` so it only fires + when the row pre-existed without a reset schedule (e.g. row created + via a different path before the proxy budget was configured). On + subsequent restarts it no-ops, so an active reset window is never + slid forward. + """ + await generate_key_helper_fn( # type: ignore + request_type="user", + table_name="user", + user_id=litellm_proxy_budget_name, + duration=None, + models=[], + aliases={}, + config={}, + spend=0, + max_budget=litellm.max_budget, + budget_duration=litellm.budget_duration, + query_type="update_data", + update_key_values={ + "max_budget": litellm.max_budget, + "budget_duration": litellm.budget_duration, + }, + ) + + # Without this, the upsert leaves budget_reset_at=NULL on rows that + # took the UPDATE path, and reset_budget_for_litellm_users never + # matches them (NULL < now() is unknown in SQL) — so the proxy-wide + # spend cap blocks forever once it's hit. + if prisma_client is not None and litellm.budget_duration is not None: + try: + await prisma_client.db.litellm_usertable.update_many( + where={ + "user_id": litellm_proxy_budget_name, + "budget_reset_at": None, + }, + data={ + "budget_reset_at": get_budget_reset_time( + budget_duration=litellm.budget_duration + ) + }, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Failed to backfill budget_reset_at on proxy admin row: %s", e + ) + @classmethod async def _warm_global_spend_cache( cls, diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 6718f52cbf1..859594f7a0b 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -1728,6 +1728,67 @@ async def test_add_proxy_budget_to_db_only_creates_user_no_keys(): assert call_args.kwargs["query_type"] == "update_data" +@pytest.mark.asyncio +async def test_add_proxy_budget_to_db_backfills_budget_reset_at(): + """ + Test that _upsert_proxy_budget_with_reset_at_backfill issues a conditional + update_many with `WHERE budget_reset_at IS NULL` to backfill the column on + rows that pre-existed without a reset schedule. Without this, the proxy + admin row stays at NULL and reset_budget_for_litellm_users never matches + it (NULL < now() is unknown in SQL), so the global proxy budget never + resets. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + import litellm + from litellm.proxy.proxy_server import ProxyStartupEvent + + litellm.budget_duration = "30d" + litellm.max_budget = 100.0 + litellm_proxy_budget_name = "litellm-proxy-budget" + + mock_prisma = MagicMock() + mock_prisma.db.litellm_usertable.update_many = AsyncMock(return_value={"count": 1}) + + mock_generate_key_helper = AsyncMock( + return_value={ + "user_id": litellm_proxy_budget_name, + "max_budget": 100.0, + "budget_duration": "30d", + "spend": 0, + "models": [], + } + ) + + with ( + patch( + "litellm.proxy.proxy_server.generate_key_helper_fn", + mock_generate_key_helper, + ), + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + ): + await ProxyStartupEvent._upsert_proxy_budget_with_reset_at_backfill( + litellm_proxy_budget_name + ) + + # Upsert ran with the configured budget + mock_generate_key_helper.assert_called_once() + + # Backfill update_many ran with the conditional WHERE + mock_prisma.db.litellm_usertable.update_many.assert_called_once() + backfill_call = mock_prisma.db.litellm_usertable.update_many.call_args + assert backfill_call.kwargs["where"]["user_id"] == litellm_proxy_budget_name + assert backfill_call.kwargs["where"]["budget_reset_at"] is None + + # The backfilled value must be a real future datetime — anything else and + # reset_budget_for_litellm_users would still skip the row. + from datetime import datetime, timezone + + backfilled_reset_at = backfill_call.kwargs["data"]["budget_reset_at"] + assert isinstance(backfilled_reset_at, datetime) + assert backfilled_reset_at > datetime.now(timezone.utc) + + @pytest.mark.asyncio async def test_custom_ui_sso_sign_in_handler_config_loading(): """ From c7739c9ed55c4d58b9191e1d1f1429091349d46d Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Sat, 9 May 2026 15:34:09 -0700 Subject: [PATCH 52/85] feat: add ability to auth to azure with token (#27556) Squash-merged by litellm-agent from shivamrawat1's PR. --- litellm/__init__.py | 1 + litellm/_redis.py | 219 +++++++++++++++++- litellm/_redis_credential_provider.py | 35 ++- .../chat/guardrail_translation/handler.py | 12 +- .../base_llm/guardrail_translation/utils.py | 15 ++ .../chat/guardrail_translation/handler.py | 24 +- .../proxy/guardrails/guardrail_registry.py | 5 + litellm/types/guardrails.py | 10 + .../test_unified_guardrail.py | 132 +++++++++++ tests/test_litellm/test_utils.py | 122 ++++++++++ .../guardrails/add_guardrail_form.tsx | 20 ++ .../guardrails/edit_guardrail_form.tsx | 23 ++ .../components/guardrails/guardrail_info.tsx | 36 +++ .../guardrail_info_helpers.test.tsx | 16 ++ .../guardrails/guardrail_info_helpers.tsx | 16 ++ .../components/guardrails/guardrail_table.tsx | 10 +- 16 files changed, 685 insertions(+), 11 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index cf05fc4c980..fd3d47ec154 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -206,6 +206,7 @@ add_user_information_to_llm_headers: Optional[bool] = ( ) store_audit_logs = False # Enterprise feature, allow users to see audit logs skip_system_message_in_guardrail: bool = False +skip_tool_message_in_guardrail: bool = False ### end of callbacks ############# email: Optional[str] = ( diff --git a/litellm/_redis.py b/litellm/_redis.py index f12afbac297..1c11ea829ba 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -19,6 +19,7 @@ import redis.asyncio as async_redis # type: ignore from litellm import get_secret, get_secret_str from litellm._redis_credential_provider import ( + AzureADCredentialProvider, GCPIAMCredentialProvider, _generate_gcp_iam_access_token, ) @@ -27,6 +28,8 @@ from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from ._logging import verbose_logger +AZURE_REDIS_SCOPE = "https://redis.azure.com/.default" + def _get_redis_kwargs(): arg_spec = inspect.getfullargspec(redis.Redis) @@ -43,6 +46,10 @@ def _get_redis_kwargs(): "redis_connect_func", "gcp_service_account", "gcp_ssl_ca_certs", + "azure_redis_ad_token", + "azure_client_id", + "azure_tenant_id", + "azure_client_secret", ] available_args = [x for x in arg_spec.args if x not in exclude_args] + include_args @@ -89,6 +96,10 @@ def _get_redis_cluster_kwargs(client=None): ) # Needed for sync clusters and IAM detection available_args.append("gcp_service_account") available_args.append("gcp_ssl_ca_certs") + available_args.append("azure_redis_ad_token") + available_args.append("azure_client_id") + available_args.append("azure_tenant_id") + available_args.append("azure_client_secret") available_args.append("max_connections") return available_args @@ -155,6 +166,125 @@ def create_gcp_iam_redis_connect_func( return iam_connect +def _build_azure_credential( + azure_client_id: Optional[str] = None, + azure_tenant_id: Optional[str] = None, + azure_client_secret: Optional[str] = None, +): + """ + Build a long-lived Azure credential object. + + Azure SDK credentials cache tokens internally and handle expiry/refresh + transparently, so this should be called once and the result reused. + """ + try: + from azure.identity import ( + ClientSecretCredential, + DefaultAzureCredential, + ManagedIdentityCredential, + ) + except ImportError: + raise ImportError( + "azure-identity is required for Azure AD Redis authentication. " + "Install it with: pip install azure-identity" + ) + + _client_id = azure_client_id or os.environ.get("AZURE_CLIENT_ID") + _tenant_id = azure_tenant_id or os.environ.get("AZURE_TENANT_ID") + _client_secret = azure_client_secret or os.environ.get("AZURE_CLIENT_SECRET") + + if _client_id and _tenant_id and _client_secret: + return ClientSecretCredential( + client_id=_client_id, + tenant_id=_tenant_id, + client_secret=_client_secret, + ) + elif _client_id: + return ManagedIdentityCredential(client_id=_client_id) + else: + return DefaultAzureCredential() + + +def _generate_azure_ad_redis_token( + azure_client_id: Optional[str] = None, + azure_tenant_id: Optional[str] = None, + azure_client_secret: Optional[str] = None, +) -> str: + """ + One-shot helper that builds a credential and fetches a single Azure AD + access token for Redis. Each call rebuilds the credential and performs a + network round-trip, so it should not be used in steady-state Redis flows + — the sync (``create_azure_ad_redis_connect_func``) and async paths + (``AzureADCredentialProvider``) keep the credential alive across + connections so the Azure SDK's internal cache + silent refresh apply. + """ + credential = _build_azure_credential( + azure_client_id=azure_client_id, + azure_tenant_id=azure_tenant_id, + azure_client_secret=azure_client_secret, + ) + token = credential.get_token(AZURE_REDIS_SCOPE) + return token.token + + +def create_azure_ad_redis_connect_func( + azure_client_id: Optional[str] = None, + azure_tenant_id: Optional[str] = None, + azure_client_secret: Optional[str] = None, +) -> Callable: + """ + Creates a custom Redis connection function for Azure AD authentication. + + Used for sync Redis clients. The credential is created once (captured by the + closure) and reused across connections — the Azure SDK handles token caching + and silent renewal internally. Only ``get_token`` is called per connection. + """ + credential = _build_azure_credential( + azure_client_id=azure_client_id, + azure_tenant_id=azure_tenant_id, + azure_client_secret=azure_client_secret, + ) + + def ad_connect(self): + """Initialize the connection and authenticate using Azure AD""" + from redis.exceptions import ( + AuthenticationError, + AuthenticationWrongNumberOfArgsError, + ) + from redis.utils import str_if_bytes + + self._parser.on_connect(self) + + access_token = credential.get_token(AZURE_REDIS_SCOPE).token + + # Only include username when explicitly set — sending AUTH "" + # is invalid for most ACL-configured Azure Redis instances. + username = os.environ.get("REDIS_USERNAME", "") + if username: + auth_args = (username, access_token) + else: + auth_args = (access_token,) + + self.send_command("AUTH", *auth_args, check_health=False) + + try: + auth_response = self.read_response() + except AuthenticationWrongNumberOfArgsError: + # Fallback: try with just the token (Redis < 6 / no ACL) + self.send_command("AUTH", access_token, check_health=False) + auth_response = self.read_response() + + if str_if_bytes(auth_response) != "OK": + raise AuthenticationError("Azure AD authentication failed for Redis") + + # Attach the live credential object so async paths can wrap it in + # AzureADCredentialProvider for refresh-aware token retrieval. The raw + # client_id/tenant_id/secret are intentionally NOT exposed here — the + # credential closure already holds them. + ad_connect._azure_credential = credential # type: ignore[attr-defined] + return ad_connect + + def get_redis_url_from_environment(): if "REDIS_URL" in os.environ: return os.environ["REDIS_URL"] @@ -179,7 +309,7 @@ def get_redis_url_from_environment(): return f"{redis_protocol}://{auth_part}{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}" -def _get_redis_client_logic(**env_overrides): +def _get_redis_client_logic(**env_overrides): # noqa: PLR0915 """ Common functionality across sync + async redis client implementations """ @@ -253,6 +383,52 @@ def _get_redis_client_logic(**env_overrides): if _gcp_ssl_ca_certs and redis_kwargs.get("ssl", False): redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs + # Handle Azure AD authentication (after GCP IAM block) + _azure_redis_ad_token = redis_kwargs.get("azure_redis_ad_token") or get_secret( + "REDIS_AZURE_AD_TOKEN" + ) + + _azure_ad_enabled = ( + _azure_redis_ad_token is not None + and str(_azure_redis_ad_token).lower() == "true" + ) + + if _azure_ad_enabled and _gcp_service_account is not None: + verbose_logger.warning( + "Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. " + "Using GCP IAM. Remove one to avoid misconfiguration." + ) + + if _azure_ad_enabled and _gcp_service_account is None: + _azure_client_id = redis_kwargs.get("azure_client_id") or get_secret_str( + "AZURE_CLIENT_ID" + ) + _azure_tenant_id = redis_kwargs.get("azure_tenant_id") or get_secret_str( + "AZURE_TENANT_ID" + ) + _azure_client_secret = redis_kwargs.get( + "azure_client_secret" + ) or get_secret_str("AZURE_CLIENT_SECRET") + + verbose_logger.debug("Setting up Azure AD authentication for Redis.") + redis_kwargs["redis_connect_func"] = create_azure_ad_redis_connect_func( + azure_client_id=_azure_client_id, + azure_tenant_id=_azure_tenant_id, + azure_client_secret=_azure_client_secret, + ) + # Marker for async paths to detect Azure AD auth. The live credential + # object is attached separately as `_azure_credential` by + # `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret + # are intentionally NOT exposed on the function to avoid leaking + # credentials via inspection or logging. + redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True # type: ignore[attr-defined] + + # Always remove Azure-specific kwargs that shouldn't be passed to Redis client + redis_kwargs.pop("azure_redis_ad_token", None) + redis_kwargs.pop("azure_client_id", None) + redis_kwargs.pop("azure_tenant_id", None) + redis_kwargs.pop("azure_client_secret", None) + if "url" in redis_kwargs and redis_kwargs["url"] is not None: # Only strip host/port/db/password when not routing to a cluster. # When startup_nodes is also present the cluster path takes priority and @@ -373,7 +549,7 @@ def get_redis_client(**env_overrides): return redis.Redis(**redis_kwargs) -def get_redis_async_client( +def get_redis_async_client( # noqa: PLR0915 connection_pool: Optional[async_redis.BlockingConnectionPool] = None, **env_overrides, ) -> Union[async_redis.Redis, async_redis.RedisCluster]: @@ -398,6 +574,14 @@ def get_redis_async_client( cluster_kwargs["credential_provider"] = GCPIAMCredentialProvider( redis_connect_func._gcp_service_account ) + # Handle Azure AD authentication for async clusters via CredentialProvider + # so the credential's internal cache + silent refresh runs per connection + # (mirrors GCP IAM above; avoids static-token-baked-in-pool expiry). + elif redis_connect_func and hasattr(redis_connect_func, "_azure_credential"): + cluster_kwargs["credential_provider"] = AzureADCredentialProvider( + redis_connect_func._azure_credential, + username=os.environ.get("REDIS_USERNAME") or None, + ) new_startup_nodes: List[ClusterNode] = [] @@ -431,6 +615,22 @@ def get_redis_async_client( # Check for Redis Sentinel if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs: return _init_async_redis_sentinel(redis_kwargs) + + # Wrap GCP / Azure AD auth in a CredentialProvider for the standard async + # Redis client. The async client doesn't support redis_connect_func, but it + # does honour credential_provider — which is called per connection, so the + # underlying SDK can refresh tokens silently before they expire. + redis_connect_func = redis_kwargs.pop("redis_connect_func", None) + if redis_connect_func and hasattr(redis_connect_func, "_azure_credential"): + redis_kwargs["credential_provider"] = AzureADCredentialProvider( + redis_connect_func._azure_credential, + username=os.environ.get("REDIS_USERNAME") or None, + ) + elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"): + redis_kwargs["credential_provider"] = GCPIAMCredentialProvider( + redis_connect_func._gcp_service_account + ) + _pretty_print_redis_config(redis_kwargs=redis_kwargs) if connection_pool is not None: @@ -464,6 +664,21 @@ def get_redis_connection_pool( redis_kwargs["max_connections"], ) return async_redis.BlockingConnectionPool.from_url(**pool_kwargs) + + # Wrap GCP / Azure AD auth in a CredentialProvider so pool-managed + # connections re-fetch tokens via the SDK's internal cache + silent refresh + # rather than reusing a single token captured at pool creation. + redis_connect_func = redis_kwargs.pop("redis_connect_func", None) + if redis_connect_func and hasattr(redis_connect_func, "_azure_credential"): + redis_kwargs["credential_provider"] = AzureADCredentialProvider( + redis_connect_func._azure_credential, + username=os.environ.get("REDIS_USERNAME") or None, + ) + elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"): + redis_kwargs["credential_provider"] = GCPIAMCredentialProvider( + redis_connect_func._gcp_service_account + ) + connection_class = async_redis.Connection if "ssl" in redis_kwargs: connection_class = async_redis.SSLConnection diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index 70725fe12c4..586b1c7716c 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -1,10 +1,13 @@ import asyncio import threading import time -from typing import Dict, Tuple +from typing import Any, Dict, Optional, Tuple, Union from redis.credentials import CredentialProvider # type: ignore[attr-defined] +# Azure AD scope for Redis Cache for Azure. +AZURE_REDIS_SCOPE = "https://redis.azure.com/.default" + # GCP IAM tokens are valid for 1 hour. Cache for 55 minutes to refresh before expiry. _GCP_IAM_TOKEN_TTL_SECONDS = 3300 @@ -101,3 +104,33 @@ class GCPIAMCredentialProvider(CredentialProvider): _get_cached_gcp_iam_token, self._gcp_service_account ) return (token,) + + +class AzureADCredentialProvider(CredentialProvider): + """ + redis.credentials.CredentialProvider implementation that supplies Azure AD + tokens for Redis authentication. + + Wraps an azure-identity credential object so the Azure SDK's internal token + cache and silent refresh are honoured on every Redis connection. This avoids + the static-token-baked-in-pool issue where pool-managed connections would + fail authentication after the initial token expired (~1 hour TTL). + """ + + def __init__(self, credential: Any, username: Optional[str] = None) -> None: + self._credential = credential + self._username = username + + def get_credentials(self) -> Union[Tuple[str], Tuple[str, str]]: + token = self._credential.get_token(AZURE_REDIS_SCOPE).token + if self._username: + return (self._username, token) + return (token,) + + async def get_credentials_async(self) -> Union[Tuple[str], Tuple[str, str]]: + token_obj = await asyncio.to_thread( + self._credential.get_token, AZURE_REDIS_SCOPE + ) + if self._username: + return (self._username, token_obj.token) + return (token_obj.token,) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 2bb82f227bb..74dadee5ecb 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -23,7 +23,9 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, openai_messages_without_system, + openai_messages_without_tool, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, @@ -108,6 +110,7 @@ class AnthropicMessagesHandler(BaseTranslation): return data skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply) + skip_tool = effective_skip_tool_message_for_guardrail(guardrail_to_apply) chat_completion_compatible_request = self._translate_to_openai(data) @@ -117,6 +120,8 @@ class AnthropicMessagesHandler(BaseTranslation): ) if skip_system: structured_messages = openai_messages_without_system(structured_messages) + if skip_tool: + structured_messages = openai_messages_without_tool(structured_messages) texts_to_check: List[str] = [] images_to_check: List[str] = [] @@ -134,6 +139,7 @@ class AnthropicMessagesHandler(BaseTranslation): images_to_check=images_to_check, task_mappings=task_mappings, skip_system_message=skip_system, + skip_tool_message=skip_tool, ) # Step 2: Apply guardrail to all texts in batch @@ -198,13 +204,17 @@ class AnthropicMessagesHandler(BaseTranslation): images_to_check: List[str], task_mappings: List[Tuple[int, Optional[int]]], skip_system_message: bool = False, + skip_tool_message: bool = False, ) -> None: """ Extract text content and images from a message. Override this method to customize text/image extraction logic. """ - if skip_system_message and str(message.get("role") or "").lower() == "system": + role = str(message.get("role") or "").lower() + if skip_system_message and role == "system": + return + if skip_tool_message and role == "tool": return content = message.get("content", None) diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index cdd2d775371..97ece6b5eab 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -14,7 +14,22 @@ def effective_skip_system_message_for_guardrail(guardrail_to_apply: Any) -> bool return bool(getattr(litellm, "skip_system_message_in_guardrail", False)) +def effective_skip_tool_message_for_guardrail(guardrail_to_apply: Any) -> bool: + per = getattr(guardrail_to_apply, "skip_tool_message_in_guardrail", None) + if per is not None: + return bool(per) + import litellm + + return bool(getattr(litellm, "skip_tool_message_in_guardrail", False)) + + def openai_messages_without_system( messages: List[AllMessageValues], ) -> List[AllMessageValues]: return [m for m in messages if str((m or {}).get("role") or "").lower() != "system"] + + +def openai_messages_without_tool( + messages: List[AllMessageValues], +) -> List[AllMessageValues]: + return [m for m in messages if str((m or {}).get("role") or "").lower() != "tool"] diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 86ca6625629..d413a244539 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -21,7 +21,9 @@ from litellm._logging import verbose_proxy_logger from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, openai_messages_without_system, + openai_messages_without_tool, ) from litellm.main import stream_chunk_builder from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam @@ -73,6 +75,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return data skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply) + skip_tool = effective_skip_tool_message_for_guardrail(guardrail_to_apply) texts_to_check: List[str] = [] images_to_check: List[str] = [] @@ -91,6 +94,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): text_task_mappings=text_task_mappings, tool_call_task_mappings=tool_call_task_mappings, skip_system_message=skip_system, + skip_tool_message=skip_tool, ) # Step 2: Apply guardrail to all texts and tool calls in batch @@ -102,11 +106,15 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs["tool_calls"] = tool_calls_to_check # type: ignore structured_messages = self.get_structured_messages(data) if structured_messages: - inputs["structured_messages"] = ( - openai_messages_without_system(structured_messages) - if skip_system - else structured_messages - ) + if skip_system: + structured_messages = openai_messages_without_system( + structured_messages + ) + if skip_tool: + structured_messages = openai_messages_without_tool( + structured_messages + ) + inputs["structured_messages"] = structured_messages # Pass tools (function definitions) to the guardrail tools = data.get("tools") if tools: @@ -176,13 +184,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation): text_task_mappings: List[Tuple[int, Optional[int]]], tool_call_task_mappings: List[Tuple[int, int]], skip_system_message: bool = False, + skip_tool_message: bool = False, ) -> None: """ Extract text content, images, and tool calls from a message. Override this method to customize text/image/tool call extraction logic. """ - if skip_system_message and str(message.get("role") or "").lower() == "system": + role = str(message.get("role") or "").lower() + if skip_system_message and role == "system": + return + if skip_tool_message and role == "tool": return content = message.get("content", None) diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 868b23756d2..838fb2e01ad 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -482,6 +482,11 @@ class InMemoryGuardrailHandler: "skip_system_message_in_guardrail", getattr(litellm_params, "skip_system_message_in_guardrail", None), ) + setattr( + custom_guardrail_callback, + "skip_tool_message_in_guardrail", + getattr(litellm_params, "skip_tool_message_in_guardrail", None), + ) parsed_guardrail = Guardrail( guardrail_id=guardrail.get("guardrail_id"), diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 04347aebe3b..751113400d3 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -633,6 +633,16 @@ class BaseLitellmParams( ), ) + skip_tool_message_in_guardrail: Optional[bool] = Field( + default=None, + description=( + "When True, unified guardrails skip tool-role messages when building " + "evaluation inputs (texts and structured_messages). When False, tool " + "messages are included even if litellm_settings sets a global skip. When " + "None, use the global litellm.skip_tool_message_in_guardrail setting." + ), + ) + # Lakera specific params category_thresholds: Optional[LakeraCategoryThresholds] = Field( default=None, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 2418d7af04b..6e027fa4941 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -8,7 +8,9 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, openai_messages_without_system, + openai_messages_without_tool, ) from litellm.llms.openai.chat.guardrail_translation.handler import ( OpenAIChatCompletionsHandler, @@ -180,6 +182,136 @@ class TestUnifiedLLMGuardrails: } assert "system" in roles + class TestSkipToolMessageForChatCompletions: + def test_openai_messages_without_tool(self): + msgs = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "content": "tool result", "tool_call_id": "call_1"}, + ] + out = openai_messages_without_tool(msgs) + assert len(out) == 2 + assert all(m["role"] != "tool" for m in out) + assert msgs[2]["content"] == "tool result" + + def test_effective_skip_tool_respects_per_guardrail_over_global( + self, monkeypatch + ): + monkeypatch.setattr( + litellm, "skip_tool_message_in_guardrail", True, raising=False + ) + + class G: + skip_tool_message_in_guardrail = False + + assert effective_skip_tool_message_for_guardrail(G()) is False + + class G2: + skip_tool_message_in_guardrail = None + + assert effective_skip_tool_message_for_guardrail(G2()) is True + + @pytest.mark.asyncio + async def test_openai_handler_skips_tool_in_guardrail_inputs(self, monkeypatch): + monkeypatch.setattr( + litellm, "skip_tool_message_in_guardrail", True, raising=False + ) + + captured = {} + + class MockGuardrail: + skip_tool_message_in_guardrail = None + + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ): + captured["inputs"] = inputs + return inputs + + data = { + "messages": [ + {"role": "user", "content": "hello"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + } + ], + }, + { + "role": "tool", + "content": "secret tool result", + "tool_call_id": "call_1", + }, + ], + "model": "gpt-4o", + } + + handler = OpenAIChatCompletionsHandler() + await handler.process_input_messages( + data=data, + guardrail_to_apply=MockGuardrail(), + litellm_logging_obj=None, + ) + + assert "secret tool result" not in captured["inputs"]["texts"] + sm = captured["inputs"].get("structured_messages") or [] + assert all(m.get("role") != "tool" for m in sm) + assert data["messages"][2]["content"] == "secret tool result" + + @pytest.mark.asyncio + async def test_openai_handler_per_guardrail_skip_tool_false_overrides_global( + self, monkeypatch + ): + monkeypatch.setattr( + litellm, "skip_tool_message_in_guardrail", True, raising=False + ) + + captured = {} + + class MockGuardrail: + skip_tool_message_in_guardrail = False + + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ): + captured["inputs"] = inputs + return inputs + + data = { + "messages": [ + {"role": "user", "content": "u"}, + {"role": "tool", "content": "tr", "tool_call_id": "call_1"}, + ], + } + + await OpenAIChatCompletionsHandler().process_input_messages( + data=data, + guardrail_to_apply=MockGuardrail(), + litellm_logging_obj=None, + ) + + assert "tr" in captured["inputs"]["texts"] + roles = { + m.get("role") + for m in (captured["inputs"].get("structured_messages") or []) + } + assert "tool" in roles + class TestAsyncPreCallHook: @pytest.mark.asyncio async def test_uses_mcp_event_type(self): diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 65305e1a81e..d07af922ea6 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -2817,6 +2817,128 @@ def test_generate_gcp_iam_access_token_import_error(): assert "pip install google-cloud-iam" in str(exc_info.value) +def test_generate_azure_ad_redis_token(): + """Test _generate_azure_ad_redis_token with mocked Azure credential.""" + from unittest.mock import Mock, patch + + expected_token = "azure-access-token-12345" + + mock_token = Mock() + mock_token.token = expected_token + + mock_credential = Mock() + mock_credential.get_token.return_value = mock_token + + mock_azure_identity = Mock() + mock_azure_identity.DefaultAzureCredential = Mock(return_value=mock_credential) + mock_azure_identity.ClientSecretCredential = Mock() + mock_azure_identity.ManagedIdentityCredential = Mock() + + with patch.dict( + "sys.modules", {"azure.identity": mock_azure_identity, "azure": Mock()} + ): + from litellm._redis import _generate_azure_ad_redis_token + + result = _generate_azure_ad_redis_token() + + assert result == expected_token + mock_credential.get_token.assert_called_once_with( + "https://redis.azure.com/.default" + ) + + +def test_generate_azure_ad_redis_token_service_principal(): + """Test _generate_azure_ad_redis_token with service principal credentials.""" + from unittest.mock import Mock, patch + + expected_token = "sp-access-token-67890" + + mock_token = Mock() + mock_token.token = expected_token + + mock_credential = Mock() + mock_credential.get_token.return_value = mock_token + + mock_client_secret_credential = Mock(return_value=mock_credential) + + mock_azure_identity = Mock() + mock_azure_identity.DefaultAzureCredential = Mock() + mock_azure_identity.ClientSecretCredential = mock_client_secret_credential + mock_azure_identity.ManagedIdentityCredential = Mock() + + with patch.dict( + "sys.modules", {"azure.identity": mock_azure_identity, "azure": Mock()} + ): + from litellm._redis import _generate_azure_ad_redis_token + + result = _generate_azure_ad_redis_token( + azure_client_id="test-client-id", + azure_tenant_id="test-tenant-id", + azure_client_secret="test-secret", + ) + + assert result == expected_token + mock_client_secret_credential.assert_called_once_with( + client_id="test-client-id", + tenant_id="test-tenant-id", + client_secret="test-secret", + ) + + +def test_generate_azure_ad_redis_token_import_error(): + """Test that _generate_azure_ad_redis_token raises ImportError when azure-identity is missing.""" + from unittest.mock import patch + from litellm._redis import _generate_azure_ad_redis_token + + with patch.dict("sys.modules", {"azure.identity": None}): + with pytest.raises(ImportError) as exc_info: + _generate_azure_ad_redis_token() + + assert "azure-identity is required" in str(exc_info.value) + + +def test_redis_client_logic_azure_ad_auth(): + """Test that _get_redis_client_logic sets up Azure AD auth when REDIS_AZURE_AD_TOKEN=true. + + Mocks ``azure.identity`` via ``sys.modules`` so the test does not require + the real ``azure-identity`` package to be installed in the CI environment. + """ + from unittest.mock import Mock, patch + + mock_credential = Mock() + mock_azure_identity = Mock() + mock_azure_identity.DefaultAzureCredential = Mock(return_value=mock_credential) + mock_azure_identity.ClientSecretCredential = Mock(return_value=mock_credential) + mock_azure_identity.ManagedIdentityCredential = Mock(return_value=mock_credential) + + with patch.dict( + "sys.modules", {"azure.identity": mock_azure_identity, "azure": Mock()} + ): + from litellm._redis import _get_redis_client_logic + + redis_kwargs = _get_redis_client_logic( + host="myredis.redis.cache.windows.net", + port="6380", + azure_redis_ad_token="true", + ssl=True, + ) + + assert "redis_connect_func" in redis_kwargs + # Marker for async paths to detect Azure AD auth + assert hasattr(redis_kwargs["redis_connect_func"], "_azure_redis_ad_token") + assert redis_kwargs["redis_connect_func"]._azure_redis_ad_token is True + # Live credential object (not raw secret) is exposed for async paths + assert hasattr(redis_kwargs["redis_connect_func"], "_azure_credential") + # Raw credentials must NOT be exposed on the function + assert not hasattr(redis_kwargs["redis_connect_func"], "_azure_client_secret") + assert not hasattr(redis_kwargs["redis_connect_func"], "_azure_client_id") + assert not hasattr(redis_kwargs["redis_connect_func"], "_azure_tenant_id") + + # Azure-specific kwargs should be removed from the dict passed to Redis + assert "azure_redis_ad_token" not in redis_kwargs + assert "azure_client_id" not in redis_kwargs + + if __name__ == "__main__": # Allow running this test file directly for debugging pytest.main([__file__, "-v"]) diff --git a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx index c91a6e85cdd..16c1c6efecd 100644 --- a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx @@ -5,6 +5,7 @@ import { createGuardrailCall, getGuardrailProviderSpecificParams, getGuardrailUI import ContentFilterConfiguration from "./content_filter/ContentFilterConfiguration"; import { choiceToSkipSystemForCreate, + choiceToSkipToolForCreate, getGuardrailProviders, guardrail_provider_map, guardrailLogoMap, @@ -188,6 +189,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a mode: preset.mode, default_on: preset.defaultOn, skip_system_message_choice: "inherit", + skip_tool_message_choice: "inherit", }; if (preset.provider === "BlockCodeExecution") { baseValues.confidence_threshold = 0.5; @@ -433,6 +435,11 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a guardrailData.litellm_params.skip_system_message_in_guardrail = skipForCreate; } + const skipToolForCreate = choiceToSkipToolForCreate(values.skip_tool_message_choice); + if (skipToolForCreate !== undefined) { + guardrailData.litellm_params.skip_tool_message_in_guardrail = skipToolForCreate; + } + // For Presidio PII, add the entity and action configurations if (values.provider === "PresidioPII" && selectedEntities.length > 0) { const piiEntitiesConfig: { [key: string]: string } = {}; @@ -804,6 +811,18 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a + + + + {/* Use the GuardrailProviderFields component to render provider-specific fields */} {!isToolPermissionProvider && !shouldRenderContentFilterConfigSettings(selectedProvider) && !shouldRenderLLMJudgeFields(selectedProvider) && ( = ({ visible, onClose, a mode: "pre_call", default_on: false, skip_system_message_choice: "inherit", + skip_tool_message_choice: "inherit", }} > {stepConfigs.map((step, index) => { diff --git a/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx b/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx index ad823df53fc..8ba9b0b312f 100644 --- a/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx @@ -6,6 +6,7 @@ import { guardrailLogoMap, getGuardrailProviders, type SkipSystemMessageChoice, + type SkipToolMessageChoice, } from "./guardrail_info_helpers"; import { getGuardrailUISettings, getGlobalLitellmHeaderName } from "../networking"; import PiiConfiguration from "./pii_configuration"; @@ -29,6 +30,7 @@ interface EditGuardrailFormProps { default_on: boolean; pii_entities_config?: { [key: string]: string }; skip_system_message_choice?: SkipSystemMessageChoice; + skip_tool_message_choice?: SkipToolMessageChoice; [key: string]: any; }; } @@ -138,6 +140,15 @@ const EditGuardrailForm: React.FC = ({ delete litellm_params.skip_system_message_in_guardrail; } + const skipToolChoice = values.skip_tool_message_choice as SkipToolMessageChoice | undefined; + if (skipToolChoice === "yes") { + litellm_params.skip_tool_message_in_guardrail = true; + } else if (skipToolChoice === "no") { + litellm_params.skip_tool_message_in_guardrail = false; + } else { + delete litellm_params.skip_tool_message_in_guardrail; + } + let guardrail_info: any = {}; // For Presidio PII, add the entity and action configurations @@ -432,6 +443,18 @@ const EditGuardrailForm: React.FC = ({ + + + + {renderProviderSpecificFields()}
diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx index 60400443d5c..53aebcff0de 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx @@ -29,7 +29,9 @@ import { getGuardrailLogoAndName, guardrail_provider_map, skipSystemMessageToChoice, + skipToolMessageToChoice, type SkipSystemMessageChoice, + type SkipToolMessageChoice, } from "./guardrail_info_helpers"; import GuardrailOptionalParams from "./guardrail_optional_params"; import GuardrailProviderFields from "./guardrail_provider_fields"; @@ -214,12 +216,16 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, if (guardrailData && form) { const lp = { ...(guardrailData.litellm_params || {}) }; delete lp.skip_system_message_in_guardrail; + delete lp.skip_tool_message_in_guardrail; form.setFieldsValue({ guardrail_name: guardrailData.guardrail_name, ...lp, skip_system_message_choice: skipSystemMessageToChoice( guardrailData.litellm_params?.skip_system_message_in_guardrail, ), + skip_tool_message_choice: skipToolMessageToChoice( + guardrailData.litellm_params?.skip_tool_message_in_guardrail, + ), guardrail_info: guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : "", // Include any optional_params if they exist ...(guardrailData.litellm_params?.optional_params && { @@ -302,6 +308,20 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, } } + const prevSkipToolChoice = skipToolMessageToChoice( + guardrailData.litellm_params?.skip_tool_message_in_guardrail, + ); + const nextSkipToolChoice = values.skip_tool_message_choice as SkipToolMessageChoice | undefined; + if (nextSkipToolChoice !== undefined && nextSkipToolChoice !== prevSkipToolChoice) { + if (nextSkipToolChoice === "inherit") { + updateData.litellm_params.skip_tool_message_in_guardrail = null; + } else if (nextSkipToolChoice === "yes") { + updateData.litellm_params.skip_tool_message_in_guardrail = true; + } else { + updateData.litellm_params.skip_tool_message_in_guardrail = false; + } + } + // Only include guardrail_info if it has changed const originalGuardrailInfo = guardrailData.guardrail_info; const newGuardrailInfo = values.guardrail_info ? JSON.parse(values.guardrail_info) : undefined; @@ -674,11 +694,15 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, ...(() => { const lp = { ...(guardrailData.litellm_params || {}) }; delete lp.skip_system_message_in_guardrail; + delete lp.skip_tool_message_in_guardrail; return lp; })(), skip_system_message_choice: skipSystemMessageToChoice( guardrailData.litellm_params?.skip_system_message_in_guardrail, ), + skip_tool_message_choice: skipToolMessageToChoice( + guardrailData.litellm_params?.skip_tool_message_in_guardrail, + ), guardrail_info: guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : "", @@ -716,6 +740,18 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, + + + + {guardrailData.litellm_params?.guardrail === "presidio" && ( <> PII Protection diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx index dfda86c1e4a..1fc62f94cf1 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx @@ -12,6 +12,8 @@ import { GuardrailProviders, skipSystemMessageToChoice, choiceToSkipSystemForCreate, + skipToolMessageToChoice, + choiceToSkipToolForCreate, } from "./guardrail_info_helpers"; describe("guardrail_info_helpers", () => { @@ -215,4 +217,18 @@ describe("guardrail_info_helpers", () => { expect(choiceToSkipSystemForCreate("no")).toBe(false); }); }); + + describe("skipToolMessageToChoice / choiceToSkipToolForCreate", () => { + it("maps API values to form choices and back for create", () => { + expect(skipToolMessageToChoice(undefined)).toBe("inherit"); + expect(skipToolMessageToChoice(null)).toBe("inherit"); + expect(skipToolMessageToChoice(true)).toBe("yes"); + expect(skipToolMessageToChoice(false)).toBe("no"); + + expect(choiceToSkipToolForCreate("inherit")).toBeUndefined(); + expect(choiceToSkipToolForCreate(undefined)).toBeUndefined(); + expect(choiceToSkipToolForCreate("yes")).toBe(true); + expect(choiceToSkipToolForCreate("no")).toBe(false); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx index ac4b787e96a..54b16b81765 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx @@ -179,3 +179,19 @@ export function choiceToSkipSystemForCreate(choice: SkipSystemMessageChoice | un if (choice === "no") return false; return undefined; } + +/** Tri-state UI value for `litellm_params.skip_tool_message_in_guardrail` (inherit = use global). */ +export type SkipToolMessageChoice = "inherit" | "yes" | "no"; + +export function skipToolMessageToChoice(v: boolean | null | undefined): SkipToolMessageChoice { + if (v === true) return "yes"; + if (v === false) return "no"; + return "inherit"; +} + +/** Create flow: omit key when inheriting global default. */ +export function choiceToSkipToolForCreate(choice: SkipToolMessageChoice | undefined): boolean | undefined { + if (choice === "yes") return true; + if (choice === "no") return false; + return undefined; +} diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx index 5bb2da78fa2..ecf6ce48fde 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx @@ -11,7 +11,12 @@ import { SortingState, useReactTable, } from "@tanstack/react-table"; -import { getGuardrailLogoAndName, guardrail_provider_map, skipSystemMessageToChoice } from "./guardrail_info_helpers"; +import { + getGuardrailLogoAndName, + guardrail_provider_map, + skipSystemMessageToChoice, + skipToolMessageToChoice, +} from "./guardrail_info_helpers"; import EditGuardrailForm from "./edit_guardrail_form"; import { Guardrail, GuardrailDefinitionLocation } from "./types"; @@ -304,6 +309,9 @@ const GuardrailTable: React.FC = ({ skip_system_message_choice: skipSystemMessageToChoice( selectedGuardrail.litellm_params?.skip_system_message_in_guardrail, ), + skip_tool_message_choice: skipToolMessageToChoice( + selectedGuardrail.litellm_params?.skip_tool_message_in_guardrail, + ), ...selectedGuardrail.guardrail_info, }} /> From 02edaef50c46dd69fc8cc5ba5dda237eef478e94 Mon Sep 17 00:00:00 2001 From: Michael-RZ-Berri Date: Sat, 9 May 2026 16:15:32 -0700 Subject: [PATCH 53/85] fix: reset org and tag budgets (#27326) * reset org budgets * reset tag budgets --------- Co-authored-by: Michael Riad Zaky --- .../proxy/common_utils/reset_budget_job.py | 152 ++++++----- .../test_proxy_budget_reset.py | 28 ++ .../common_utils/test_reset_budget_job.py | 253 ++++++++++++++++++ 3 files changed, 363 insertions(+), 70 deletions(-) diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 0928ce914da..d4c5d76ac13 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -2,7 +2,7 @@ import asyncio import json import time from datetime import datetime, timezone -from typing import Any, List, Literal, Optional, Union +from typing import Any, Callable, List, Literal, Optional, Union import litellm from litellm._logging import verbose_proxy_logger @@ -83,93 +83,97 @@ class ResetBudgetJob: "Failed to reset spend counter %s: %s", counter_key, e ) + async def _cascade_reset_spend_for_budget_link( + self, + budgets_to_reset: List[LiteLLM_BudgetTableFull], + table: Any, + counter_key_fn: Callable[[Any], str], + log_subject: str, + extra_where: Optional[dict] = None, + ): + """ + Generic cascade: zero spend on rows whose budget_id is in the reset set. + """ + budget_ids = [b.budget_id for b in budgets_to_reset if b.budget_id is not None] + if not budget_ids: + return + + where: dict = {"budget_id": {"in": budget_ids}} + if extra_where: + where.update(extra_where) + + try: + rows = await table.find_many(where=where) + except Exception as e: + rows = [] + verbose_proxy_logger.warning( + "Failed to fetch %s for counter invalidation: %s", log_subject, e + ) + + update_result = await table.update_many(where=where, data={"spend": 0}) + + for row in rows: + await self._invalidate_spend_counter(counter_key_fn(row)) + + return update_result + async def reset_budget_for_litellm_team_members( self, budgets_to_reset: List[LiteLLM_BudgetTableFull] ): """ Resets the budget for all LiteLLM Team Members if their budget has expired """ - budget_ids = [ - budget.budget_id - for budget in budgets_to_reset - if budget.budget_id is not None - ] - - try: - memberships = await self.prisma_client.db.litellm_teammembership.find_many( - where={"budget_id": {"in": budget_ids}} - ) - except Exception as e: - memberships = [] - verbose_proxy_logger.warning( - "Failed to fetch team memberships for counter invalidation: %s", e - ) - - update_result = await self.prisma_client.db.litellm_teammembership.update_many( - where={"budget_id": {"in": budget_ids}}, - data={ - "spend": 0, - }, + return await self._cascade_reset_spend_for_budget_link( + budgets_to_reset=budgets_to_reset, + table=self.prisma_client.db.litellm_teammembership, + counter_key_fn=lambda m: f"spend:team_member:{m.user_id}:{m.team_id}", + log_subject="team memberships", ) - for m in memberships: - await self._invalidate_spend_counter( - f"spend:team_member:{m.user_id}:{m.team_id}" - ) - - return update_result - async def reset_budget_for_keys_linked_to_budgets( self, budgets_to_reset: List[LiteLLM_BudgetTableFull] ): """ Resets the spend for keys linked to budget tiers that are being reset. - This handles keys that have budget_id but no budget_duration set on the key - itself. Keys with budget_id rely on their linked budget tier's reset schedule - rather than having their own budget_duration. - - Keys that have their own budget_duration are already handled by - reset_budget_for_litellm_keys() and are excluded here to avoid - double-resetting. + Excludes keys with their own budget_duration; those are reset by + reset_budget_for_litellm_keys() to avoid double-resetting. """ - budget_ids = [ - budget.budget_id - for budget in budgets_to_reset - if budget.budget_id is not None - ] - if not budget_ids: - return - - where_clause: dict = { - "budget_id": {"in": budget_ids}, - "budget_duration": None, # only keys without their own reset schedule - "spend": {"gt": 0}, # only reset keys that have accumulated spend - } - - try: - keys = await self.prisma_client.db.litellm_verificationtoken.find_many( - where=where_clause - ) - except Exception as e: - keys = [] - verbose_proxy_logger.warning( - "Failed to fetch keys for counter invalidation: %s", e - ) - - update_result = ( - await self.prisma_client.db.litellm_verificationtoken.update_many( - where=where_clause, - data={ - "spend": 0, - }, - ) + return await self._cascade_reset_spend_for_budget_link( + budgets_to_reset=budgets_to_reset, + table=self.prisma_client.db.litellm_verificationtoken, + counter_key_fn=lambda k: f"spend:key:{k.token}", + log_subject="keys", + extra_where={"budget_duration": None, "spend": {"gt": 0}}, ) - for k in keys: - await self._invalidate_spend_counter(f"spend:key:{k.token}") + async def reset_budget_for_orgs_linked_to_budgets( + self, budgets_to_reset: List[LiteLLM_BudgetTableFull] + ): + """ + Resets the spend for orgs linked to budget tiers that are being reset. + """ + return await self._cascade_reset_spend_for_budget_link( + budgets_to_reset=budgets_to_reset, + table=self.prisma_client.db.litellm_organizationtable, + counter_key_fn=lambda o: f"spend:org:{o.organization_id}", + log_subject="orgs", + extra_where={"spend": {"gt": 0}}, + ) - return update_result + async def reset_budget_for_tags_linked_to_budgets( + self, budgets_to_reset: List[LiteLLM_BudgetTableFull] + ): + """ + Resets the spend for tags linked to budget tiers that are being reset. + """ + return await self._cascade_reset_spend_for_budget_link( + budgets_to_reset=budgets_to_reset, + table=self.prisma_client.db.litellm_tagtable, + counter_key_fn=lambda t: f"spend:tag:{t.tag_name}", + log_subject="tags", + extra_where={"spend": {"gt": 0}}, + ) async def reset_budget_for_litellm_budget_table(self): """ @@ -237,6 +241,14 @@ class ResetBudgetJob: budgets_to_reset=budgets_to_reset ) + await self.reset_budget_for_orgs_linked_to_budgets( + budgets_to_reset=budgets_to_reset + ) + + await self.reset_budget_for_tags_linked_to_budgets( + budgets_to_reset=budgets_to_reset + ) + if endusers_to_reset is not None and len(endusers_to_reset) > 0: for enduser in endusers_to_reset: try: diff --git a/tests/litellm_utils_tests/test_proxy_budget_reset.py b/tests/litellm_utils_tests/test_proxy_budget_reset.py index a64c6c7aa36..6240bedd3e6 100644 --- a/tests/litellm_utils_tests/test_proxy_budget_reset.py +++ b/tests/litellm_utils_tests/test_proxy_budget_reset.py @@ -233,6 +233,12 @@ async def test_reset_budget_endusers_partial_failure(): prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( return_value={"count": 0} ) + # Mock db.litellm_organizationtable.update_many (used by reset_budget_for_orgs_linked_to_budgets) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 0} + ) + # Mock db.litellm_tagtable.update_many (used by reset_budget_for_tags_linked_to_budgets) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 0}) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() @@ -400,6 +406,12 @@ async def test_reset_budget_continues_other_categories_on_failure(): prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( return_value={"count": 0} ) + # Mock db.litellm_organizationtable.update_many (used by reset_budget_for_orgs_linked_to_budgets) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 0} + ) + # Mock db.litellm_tagtable.update_many (used by reset_budget_for_tags_linked_to_budgets) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 0}) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() @@ -884,6 +896,12 @@ async def test_service_logger_endusers_success(): prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( return_value={"count": 0} ) + # Mock db.litellm_organizationtable.update_many (used by reset_budget_for_orgs_linked_to_budgets) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 0} + ) + # Mock db.litellm_tagtable.update_many (used by reset_budget_for_tags_linked_to_budgets) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 0}) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() @@ -966,6 +984,12 @@ async def test_service_logger_endusers_failure(): prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( return_value={"count": 0} ) + # Mock db.litellm_organizationtable.update_many (used by reset_budget_for_orgs_linked_to_budgets) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 0} + ) + # Mock db.litellm_tagtable.update_many (used by reset_budget_for_tags_linked_to_budgets) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 0}) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() @@ -1060,6 +1084,10 @@ async def test_reset_budget_for_litellm_team_members_called(): prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( return_value={"count": 0} ) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 0} + ) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 0}) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py index 5c86f9057a1..82511fbc55e 100644 --- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py +++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py @@ -39,6 +39,46 @@ class MockLiteLLMVerificationToken: return {"count": 1} +class MockLiteLLMOrganizationTable: + def __init__(self): + self.update_many_calls: List[Dict[str, Any]] = [] + self.find_many_calls: List[Dict[str, Any]] = [] + self._find_many_results: List[Any] = [] + + def set_find_many_results(self, results: List[Any]): + self._find_many_results = results + + async def find_many(self, where: Dict[str, Any]) -> List[Any]: + self.find_many_calls.append({"where": where}) + return self._find_many_results + + async def update_many( + self, where: Dict[str, Any], data: Dict[str, Any] + ) -> Dict[str, Any]: + self.update_many_calls.append({"where": where, "data": data}) + return {"count": 1} + + +class MockLiteLLMTagTable: + def __init__(self): + self.update_many_calls: List[Dict[str, Any]] = [] + self.find_many_calls: List[Dict[str, Any]] = [] + self._find_many_results: List[Any] = [] + + def set_find_many_results(self, results: List[Any]): + self._find_many_results = results + + async def find_many(self, where: Dict[str, Any]) -> List[Any]: + self.find_many_calls.append({"where": where}) + return self._find_many_results + + async def update_many( + self, where: Dict[str, Any], data: Dict[str, Any] + ) -> Dict[str, Any]: + self.update_many_calls.append({"where": where, "data": data}) + return {"count": 1} + + class MockLiteLLMEndUserTable: def __init__(self): self.find_many_calls: List[Dict[str, Any]] = [] @@ -57,6 +97,8 @@ class MockDB: self.litellm_teammembership = MockLiteLLMTeamMembership() self.litellm_verificationtoken = MockLiteLLMVerificationToken() self.litellm_endusertable = MockLiteLLMEndUserTable() + self.litellm_organizationtable = MockLiteLLMOrganizationTable() + self.litellm_tagtable = MockLiteLLMTagTable() class MockPrismaClient: @@ -459,6 +501,100 @@ def test_reset_budget_for_keys_linked_to_budgets_empty( assert len(calls) == 0 +def test_reset_budget_for_orgs_linked_to_budgets(reset_budget_job, mock_prisma_client): + """ + Test that when a budget tier is reset, orgs linked to that budget + (via budget_id) also get their spend reset. + """ + now = datetime.now(timezone.utc) + + test_budget = type( + "LiteLLM_BudgetTableFull", + (), + { + "max_budget": 100.0, + "budget_duration": "30d", + "budget_reset_at": now - timedelta(hours=1), + "budget_id": "30d-org-budget", + "created_at": now - timedelta(days=30), + }, + ) + + asyncio.run( + reset_budget_job.reset_budget_for_orgs_linked_to_budgets( + budgets_to_reset=[test_budget] + ) + ) + + calls = mock_prisma_client.db.litellm_organizationtable.update_many_calls + assert len(calls) == 1 + call = calls[0] + assert call["where"]["budget_id"] == {"in": ["30d-org-budget"]} + assert call["where"]["spend"] == {"gt": 0} + assert call["data"]["spend"] == 0 + + +def test_reset_budget_for_orgs_linked_to_budgets_empty( + reset_budget_job, mock_prisma_client +): + """ + Test that when there are no budgets to reset, no update is performed + on the organization table. + """ + asyncio.run( + reset_budget_job.reset_budget_for_orgs_linked_to_budgets(budgets_to_reset=[]) + ) + calls = mock_prisma_client.db.litellm_organizationtable.update_many_calls + assert len(calls) == 0 + + +def test_reset_budget_for_tags_linked_to_budgets(reset_budget_job, mock_prisma_client): + """ + Test that when a budget tier is reset, tags linked to that budget + (via budget_id) also get their spend reset. + """ + now = datetime.now(timezone.utc) + + test_budget = type( + "LiteLLM_BudgetTableFull", + (), + { + "max_budget": 50.0, + "budget_duration": "30d", + "budget_reset_at": now - timedelta(hours=1), + "budget_id": "30d-tag-budget", + "created_at": now - timedelta(days=30), + }, + ) + + asyncio.run( + reset_budget_job.reset_budget_for_tags_linked_to_budgets( + budgets_to_reset=[test_budget] + ) + ) + + calls = mock_prisma_client.db.litellm_tagtable.update_many_calls + assert len(calls) == 1 + call = calls[0] + assert call["where"]["budget_id"] == {"in": ["30d-tag-budget"]} + assert call["where"]["spend"] == {"gt": 0} + assert call["data"]["spend"] == 0 + + +def test_reset_budget_for_tags_linked_to_budgets_empty( + reset_budget_job, mock_prisma_client +): + """ + Test that when there are no budgets to reset, no update is performed + on the tag table. + """ + asyncio.run( + reset_budget_job.reset_budget_for_tags_linked_to_budgets(budgets_to_reset=[]) + ) + calls = mock_prisma_client.db.litellm_tagtable.update_many_calls + assert len(calls) == 0 + + @pytest.mark.parametrize( "budget_duration, expected_day, expected_month", [ @@ -618,6 +754,75 @@ def test_budget_table_reset_also_resets_linked_keys( assert calls[0]["data"]["spend"] == 0 +def test_budget_table_reset_also_resets_linked_orgs( + reset_budget_job, mock_prisma_client +): + """ + Integration-style test: when reset_budget_for_litellm_budget_table runs, + it should also reset spend for orgs linked to the expiring budget tiers + (in addition to end-users, team members, and keys). + """ + now = datetime.now(timezone.utc) + + test_budget = type( + "LiteLLM_BudgetTableFull", + (), + { + "max_budget": 100.0, + "budget_duration": "30d", + "budget_reset_at": now - timedelta(hours=1), + "budget_id": "30d-org-budget", + "created_at": now - timedelta(days=30), + }, + ) + + mock_prisma_client.data["budget"] = [test_budget] + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + calls = mock_prisma_client.db.litellm_organizationtable.update_many_calls + assert len(calls) == 1, ( + "Expected reset_budget_for_litellm_budget_table to also reset orgs " + f"linked to expiring budgets, but got {len(calls)} update_many calls" + ) + assert calls[0]["where"]["budget_id"] == {"in": ["30d-org-budget"]} + assert calls[0]["data"]["spend"] == 0 + + +def test_budget_table_reset_also_resets_linked_tags( + reset_budget_job, mock_prisma_client +): + """ + Integration-style test: when reset_budget_for_litellm_budget_table runs, + it should also reset spend for tags linked to the expiring budget tiers. + """ + now = datetime.now(timezone.utc) + + test_budget = type( + "LiteLLM_BudgetTableFull", + (), + { + "max_budget": 50.0, + "budget_duration": "30d", + "budget_reset_at": now - timedelta(hours=1), + "budget_id": "30d-tag-budget", + "created_at": now - timedelta(days=30), + }, + ) + + mock_prisma_client.data["budget"] = [test_budget] + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + calls = mock_prisma_client.db.litellm_tagtable.update_many_calls + assert len(calls) == 1, ( + "Expected reset_budget_for_litellm_budget_table to also reset tags " + f"linked to expiring budgets, but got {len(calls)} update_many calls" + ) + assert calls[0]["where"]["budget_id"] == {"in": ["30d-tag-budget"]} + assert calls[0]["data"]["spend"] == 0 + + def test_reset_budget_resets_endusers_with_null_budget_id( reset_budget_job, mock_prisma_client ): @@ -1205,3 +1410,51 @@ def test_reset_budget_for_keys_linked_to_budgets_invalidates_redis_counter(monke counter_cache.in_memory_cache.set_cache.assert_any_call( key="spend:key:sk-linked", value=0.0, ttl=60 ) + + +def test_reset_budget_for_orgs_linked_to_budgets_invalidates_redis_counter(monkeypatch): + """Resetting orgs via budget tier must clear each linked org's counter.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_org = type("Org", (), {"organization_id": "org-acme"}) + + prisma_client = MagicMock() + prisma_client.db.litellm_organizationtable.find_many = AsyncMock( + return_value=[linked_org] + ) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 1} + ) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_orgs_linked_to_budgets([expired_budget])) + + counter_cache.in_memory_cache.set_cache.assert_any_call( + key="spend:org:org-acme", value=0.0, ttl=60 + ) + counter_cache.redis_cache.async_set_cache.assert_any_await( + key="spend:org:org-acme", value=0.0, ttl=60 + ) + + +def test_reset_budget_for_tags_linked_to_budgets_invalidates_redis_counter(monkeypatch): + """Resetting tags via budget tier must clear each linked tag's counter.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_tag = type("Tag", (), {"tag_name": "tenant-42"}) + + prisma_client = MagicMock() + prisma_client.db.litellm_tagtable.find_many = AsyncMock(return_value=[linked_tag]) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 1}) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_tags_linked_to_budgets([expired_budget])) + + counter_cache.in_memory_cache.set_cache.assert_any_call( + key="spend:tag:tenant-42", value=0.0, ttl=60 + ) + counter_cache.redis_cache.async_set_cache.assert_any_await( + key="spend:tag:tenant-42", value=0.0, ttl=60 + ) From 9f68d2bb774b1d68696ae15460f3bd1ec74167bb Mon Sep 17 00:00:00 2001 From: oss-agent-shin Date: Sat, 9 May 2026 17:18:55 -0700 Subject: [PATCH 54/85] Fix: tag budget reset must drop stale management-cache entry (#27568) Squash-merged by litellm-agent from oss-agent-shin's PR. --- AGENTS.md | 21 +- CLAUDE.md | 2 +- deploy/Dockerfile.ghcr_base | 18 - deploy/kubernetes/kub.yaml | 56 - deploy/kubernetes/service.yaml | 12 - dev_config.yaml | 13 - docker/Dockerfile.alpine | 68 - docker/Dockerfile.custom_ui | 86 - docker/Dockerfile.dev | 121 - docker/Dockerfile.health_check | 30 - index.yaml | 108 - litellm-js/proxy/.npmrc | 5 - litellm-js/proxy/README.md | 8 - litellm-js/proxy/package-lock.json | 2054 ----------------- litellm-js/proxy/package.json | 14 - litellm-js/proxy/src/index.ts | 59 - litellm-js/proxy/tsconfig.json | 17 - litellm-js/proxy/wrangler.toml | 18 - litellm-js/spend-logs/.npmrc | 5 - litellm-js/spend-logs/Dockerfile | 26 - litellm-js/spend-logs/README.md | 8 - litellm-js/spend-logs/package-lock.json | 597 ----- litellm-js/spend-logs/package.json | 13 - litellm-js/spend-logs/schema.prisma | 29 - litellm-js/spend-logs/src/_types.ts | 32 - litellm-js/spend-logs/src/index.ts | 84 - litellm-js/spend-logs/tsconfig.json | 13 - litellm/proxy/auth/auth_checks.py | 6 +- litellm/proxy/auth/auth_exception_handler.py | 2 +- litellm/proxy/auth/user_api_key_auth.py | 6 +- .../proxy/common_utils/reset_budget_job.py | 194 +- litellm/proxy/proxy_server.py | 74 +- pyproject.toml | 2 +- .../test_proxy_budget_reset.py | 28 + tests/otel_tests/test_e2e_budgeting.py | 4 +- tests/otel_tests/test_e2e_model_access.py | 4 +- .../proxy/auth/test_auth_checks.py | 49 + .../proxy/auth/test_auth_exception_handler.py | 1 + .../proxy/auth/test_user_api_key_auth.py | 23 + .../common_utils/test_reset_budget_job.py | 350 ++- tests/test_litellm/proxy/test_proxy_server.py | 61 + tests/test_openai_endpoints.py | 2 +- tests/test_users.py | 4 +- uv.lock | 2 +- 44 files changed, 727 insertions(+), 3602 deletions(-) delete mode 100644 deploy/Dockerfile.ghcr_base delete mode 100644 deploy/kubernetes/kub.yaml delete mode 100644 deploy/kubernetes/service.yaml delete mode 100644 dev_config.yaml delete mode 100644 docker/Dockerfile.alpine delete mode 100644 docker/Dockerfile.custom_ui delete mode 100644 docker/Dockerfile.dev delete mode 100644 docker/Dockerfile.health_check delete mode 100644 index.yaml delete mode 100644 litellm-js/proxy/.npmrc delete mode 100644 litellm-js/proxy/README.md delete mode 100644 litellm-js/proxy/package-lock.json delete mode 100644 litellm-js/proxy/package.json delete mode 100644 litellm-js/proxy/src/index.ts delete mode 100644 litellm-js/proxy/tsconfig.json delete mode 100644 litellm-js/proxy/wrangler.toml delete mode 100644 litellm-js/spend-logs/.npmrc delete mode 100644 litellm-js/spend-logs/Dockerfile delete mode 100644 litellm-js/spend-logs/README.md delete mode 100644 litellm-js/spend-logs/package-lock.json delete mode 100644 litellm-js/spend-logs/package.json delete mode 100644 litellm-js/spend-logs/schema.prisma delete mode 100644 litellm-js/spend-logs/src/_types.ts delete mode 100644 litellm-js/spend-logs/src/index.ts delete mode 100644 litellm-js/spend-logs/tsconfig.json diff --git a/AGENTS.md b/AGENTS.md index 4bdbf26ae9d..e99bf79d783 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -241,10 +241,27 @@ When opening issues or pull requests, follow these templates: ### Running the proxy server -Start the proxy with a config file: +Create a minimal config file and start the proxy: + +```yaml +# config.yaml +model_list: + - model_name: fake-openai-endpoint + litellm_params: + model: openai/fake-model + api_key: fake-key + api_base: https://fake-api.example.com + +general_settings: + master_key: sk-1234 + +litellm_settings: + drop_params: True + telemetry: False +``` ```bash -uv run litellm --config dev_config.yaml --port 4000 +uv run litellm --config config.yaml --port 4000 ``` The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package. diff --git a/CLAUDE.md b/CLAUDE.md index 71e5af28ee7..938801df7c7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -146,7 +146,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components: - **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets. - **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields. - **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries. -- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`. +- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`. ### Setup Wizard (`litellm/setup_wizard.py`) - The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI). diff --git a/deploy/Dockerfile.ghcr_base b/deploy/Dockerfile.ghcr_base deleted file mode 100644 index 66e64e5b774..00000000000 --- a/deploy/Dockerfile.ghcr_base +++ /dev/null @@ -1,18 +0,0 @@ -# Use the provided base image -FROM ghcr.io/berriai/litellm:main-latest@sha256:7c311546c25e7bb6e8cafede9fcd3d0d622ac636b5c9418befaa32e85dfb0186 - -# Set the working directory to /app -WORKDIR /app - -# Copy the configuration file into the container at /app -COPY config.yaml . - -# Make sure your docker/entrypoint.sh is executable -# Convert Windows line endings to Unix -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh - -# Expose the necessary port -EXPOSE 4000/tcp - -# Override the CMD instruction with your desired command and arguments -CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug", "--run_gunicorn"] diff --git a/deploy/kubernetes/kub.yaml b/deploy/kubernetes/kub.yaml deleted file mode 100644 index d5ba500d8f0..00000000000 --- a/deploy/kubernetes/kub.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: litellm-deployment -spec: - replicas: 3 - selector: - matchLabels: - app: litellm - template: - metadata: - labels: - app: litellm - spec: - containers: - - name: litellm-container - image: ghcr.io/berriai/litellm:main-latest - imagePullPolicy: Always - env: - - name: AZURE_API_KEY - value: "d6f****" - - name: AZURE_API_BASE - value: "https://openai" - - name: LITELLM_MASTER_KEY - value: "sk-1234" - - name: DATABASE_URL - value: "postgresql://ishaan*********" - args: - - "--config" - - "/app/proxy_config.yaml" # Update the path to mount the config file - volumeMounts: # Define volume mount for proxy_config.yaml - - name: config-volume - mountPath: /app - readOnly: true - livenessProbe: - httpGet: - path: /health/liveliness - port: 4000 - initialDelaySeconds: 120 - periodSeconds: 15 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: /health/readiness - port: 4000 - initialDelaySeconds: 120 - periodSeconds: 15 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 10 - volumes: # Define volume to mount proxy_config.yaml - - name: config-volume - configMap: - name: litellm-config diff --git a/deploy/kubernetes/service.yaml b/deploy/kubernetes/service.yaml deleted file mode 100644 index 4751c837254..00000000000 --- a/deploy/kubernetes/service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: litellm-service -spec: - selector: - app: litellm - ports: - - protocol: TCP - port: 4000 - targetPort: 4000 - type: LoadBalancer \ No newline at end of file diff --git a/dev_config.yaml b/dev_config.yaml deleted file mode 100644 index 64e3c14703e..00000000000 --- a/dev_config.yaml +++ /dev/null @@ -1,13 +0,0 @@ -model_list: - - model_name: fake-openai-endpoint - litellm_params: - model: openai/fake-model - api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - -general_settings: - master_key: sk-1234 - -litellm_settings: - drop_params: True - telemetry: False diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine deleted file mode 100644 index 5de588cf4e4..00000000000 --- a/docker/Dockerfile.alpine +++ /dev/null @@ -1,68 +0,0 @@ -# Base image for building -ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856 - -# Runtime image -ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856 -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a - -FROM $UV_IMAGE AS uvbin - -FROM $LITELLM_BUILD_IMAGE AS builder - -WORKDIR /app - -COPY --from=uvbin /uv /usr/local/bin/uv -COPY --from=uvbin /uvx /usr/local/bin/uvx - -RUN apk add --no-cache gcc python3-dev musl-dev nodejs npm libsndfile - -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -# Copy dependency metadata first for layer caching -COPY pyproject.toml uv.lock ./ -COPY enterprise/pyproject.toml enterprise/ -COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ - -# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) -RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python3 - -# Copy full source tree -COPY . . - -# Install project and workspace packages (fast - deps already cached) -RUN uv sync --frozen --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python3 - -RUN prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -FROM $LITELLM_RUNTIME_IMAGE AS runtime - -RUN apk upgrade --no-cache && apk add --no-cache libsndfile nodejs npm - -WORKDIR /app -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -COPY --from=builder /app /app - -EXPOSE 4000/tcp - -ENTRYPOINT ["docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] diff --git a/docker/Dockerfile.custom_ui b/docker/Dockerfile.custom_ui deleted file mode 100644 index cc44893bf92..00000000000 --- a/docker/Dockerfile.custom_ui +++ /dev/null @@ -1,86 +0,0 @@ -# Use the provided base image -# NOTE: This is a dev/branch-specific tag. Update digest when the base image is rebuilt. -FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev - -# Set the working directory to /app -WORKDIR /app - -# Install Node.js and npm (adjust version as needed) -RUN apt-get update && apt-get upgrade -y \ - libxml2 \ - libexpat1 \ - openssl \ - libssl3 \ - git \ - libkrb5-3 \ - libglib2.0-0 \ - wget \ - libaom3 \ - libxslt1.1 \ - libgnutls30 \ - libc6 && \ - apt-get install -y --no-install-recommends nodejs npm && \ - npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ - GLOBAL="$(npm root -g)" && \ - find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ - done && \ - find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \ - sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \ - npm cache clean --force && \ - apt-get purge -y npm - -# Copy the UI source into the container -COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard - -# Set an environment variable for UI_BASE_PATH -# This can be overridden at build time -# set UI_BASE_PATH to "/ui" -ENV UI_BASE_PATH="/prod/ui" - -# Build the UI with the specified UI_BASE_PATH -WORKDIR /app/ui/litellm-dashboard -RUN npm ci -RUN UI_BASE_PATH=$UI_BASE_PATH npm run build - -# Create the destination directory -RUN mkdir -p /app/litellm/proxy/_experimental/out - -# Move the built files to the appropriate location -# Assuming the build output is in ./out directory -RUN rm -rf /app/litellm/proxy/_experimental/out/* && \ - mv ./out/* /app/litellm/proxy/_experimental/out/ - -# Switch back to the main app directory -WORKDIR /app - -# Make sure your docker/entrypoint.sh is executable -# Convert Windows line endings to Unix for entrypoint scripts -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh -RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -# Run as non-root user -RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser \ - && chown -R appuser:appuser /app -USER appuser - -# Expose the necessary port -EXPOSE 4000/tcp - -HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ - CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health')"] - -# Override the CMD instruction with your desired command and arguments -CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"] \ No newline at end of file diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev deleted file mode 100644 index ebc92a22d50..00000000000 --- a/docker/Dockerfile.dev +++ /dev/null @@ -1,121 +0,0 @@ -# Base image for building -ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d - -# Runtime image -ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a - -FROM $UV_IMAGE AS uvbin - -FROM $LITELLM_BUILD_IMAGE AS builder - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /usr/local/bin/uv -COPY --from=uvbin /uvx /usr/local/bin/uvx - -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc \ - g++ \ - python3-dev \ - libssl-dev \ - pkg-config \ - nodejs \ - npm \ - && rm -rf /var/lib/apt/lists/* - -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -# Copy dependency metadata first for layer caching -COPY pyproject.toml uv.lock ./ -COPY enterprise/pyproject.toml enterprise/ -COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ - -# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) -RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python - -# Copy full source tree -COPY . . - -# Build Admin UI before final sync -RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh - -# Install project and workspace packages (fast - deps already cached) -RUN uv sync --frozen --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python - -RUN prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -FROM $LITELLM_RUNTIME_IMAGE AS runtime - -USER root - -RUN apt-get update && apt-get upgrade -y \ - libxml2 \ - libexpat1 \ - openssl \ - libssl3 \ - git \ - libkrb5-3 \ - libglib2.0-0 \ - wget \ - libaom3 \ - libxslt1.1 \ - libgnutls30 \ - libc6 \ - && apt-get install -y --no-install-recommends \ - libssl3 \ - libatomic1 \ - nodejs \ - npm \ - && rm -rf /var/lib/apt/lists/* \ - && npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \ - && GLOBAL="$(npm root -g)" \ - && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ - done \ - && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \ - sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \ - && npm cache clean --force \ - && apt-get purge -y npm - -WORKDIR /app -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -COPY --from=builder /app /app - -EXPOSE 4000/tcp - -ENTRYPOINT ["docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] diff --git a/docker/Dockerfile.health_check b/docker/Dockerfile.health_check deleted file mode 100644 index a2e5cb9f71f..00000000000 --- a/docker/Dockerfile.health_check +++ /dev/null @@ -1,30 +0,0 @@ -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a -FROM $UV_IMAGE AS uvbin - -FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d - -WORKDIR /app - -# Copy the uv binary and the health check script. -COPY --from=uvbin /uv /usr/local/bin/uv -COPY pyproject.toml uv.lock /app/ -COPY scripts/health_check/health_check_client.py /app/health_check_client.py - -# Resolve and install the health-check dependencies from the project lockfile -# so the runtime image stays self-contained and reproducible. -RUN uv export --frozen --no-default-groups --only-group healthcheck --no-emit-project --no-hashes --output-file /tmp/health-check-requirements.txt \ - && uv pip install --system -r /tmp/health-check-requirements.txt \ - && rm /tmp/health-check-requirements.txt \ - && rm /app/pyproject.toml /app/uv.lock \ - && chmod +x /app/health_check_client.py - -# Run as non-root user -RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser -USER appuser - -# Health check -HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \ - CMD ["python", "/app/health_check_client.py", "--help"] - -# Set entrypoint -ENTRYPOINT ["python", "/app/health_check_client.py"] diff --git a/index.yaml b/index.yaml deleted file mode 100644 index 9b2461c36b5..00000000000 --- a/index.yaml +++ /dev/null @@ -1,108 +0,0 @@ -apiVersion: v1 -entries: - litellm-helm: - - apiVersion: v2 - appVersion: v1.43.18 - created: "2024-08-19T23:58:25.331689+08:00" - dependencies: - - condition: db.deployStandalone - name: postgresql - repository: oci://registry-1.docker.io/bitnamicharts - version: '>=13.3.0' - - condition: redis.enabled - name: redis - repository: oci://registry-1.docker.io/bitnamicharts - version: '>=18.0.0' - description: Call all LLM APIs using the OpenAI format - digest: 0411df3dc42868be8af3ad3e00cb252790e6bd7ad15f5b77f1ca5214573a8531 - name: litellm-helm - type: application - urls: - - https://berriai.github.io/litellm/litellm-helm-0.2.3.tgz - version: 0.2.3 - postgresql: - - annotations: - category: Database - images: | - - name: os-shell - image: docker.io/bitnami/os-shell:12-debian-12-r16 - - name: postgres-exporter - image: docker.io/bitnami/postgres-exporter:0.15.0-debian-12-r14 - - name: postgresql - image: docker.io/bitnami/postgresql:16.2.0-debian-12-r6 - licenses: Apache-2.0 - apiVersion: v2 - appVersion: 16.2.0 - created: "2024-08-19T23:58:25.335716+08:00" - dependencies: - - name: common - repository: oci://registry-1.docker.io/bitnamicharts - tags: - - bitnami-common - version: 2.x.x - description: PostgreSQL (Postgres) is an open source object-relational database - known for reliability and data integrity. ACID-compliant, it supports foreign - keys, joins, views, triggers and stored procedures. - digest: 3c8125526b06833df32e2f626db34aeaedb29d38f03d15349db6604027d4a167 - home: https://bitnami.com - icon: https://bitnami.com/assets/stacks/postgresql/img/postgresql-stack-220x234.png - keywords: - - postgresql - - postgres - - database - - sql - - replication - - cluster - maintainers: - - name: VMware, Inc. - url: https://github.com/bitnami/charts - name: postgresql - sources: - - https://github.com/bitnami/charts/tree/main/bitnami/postgresql - urls: - - https://berriai.github.io/litellm/charts/postgresql-14.3.1.tgz - version: 14.3.1 - redis: - - annotations: - category: Database - images: | - - name: kubectl - image: docker.io/bitnami/kubectl:1.29.2-debian-12-r3 - - name: os-shell - image: docker.io/bitnami/os-shell:12-debian-12-r16 - - name: redis - image: docker.io/bitnami/redis:7.2.4-debian-12-r9 - - name: redis-exporter - image: docker.io/bitnami/redis-exporter:1.58.0-debian-12-r4 - - name: redis-sentinel - image: docker.io/bitnami/redis-sentinel:7.2.4-debian-12-r7 - licenses: Apache-2.0 - apiVersion: v2 - appVersion: 7.2.4 - created: "2024-08-19T23:58:25.339392+08:00" - dependencies: - - name: common - repository: oci://registry-1.docker.io/bitnamicharts - tags: - - bitnami-common - version: 2.x.x - description: Redis(R) is an open source, advanced key-value store. It is often - referred to as a data structure server since keys can contain strings, hashes, - lists, sets and sorted sets. - digest: b2fa1835f673a18002ca864c54fadac3c33789b26f6c5e58e2851b0b14a8f984 - home: https://bitnami.com - icon: https://bitnami.com/assets/stacks/redis/img/redis-stack-220x234.png - keywords: - - redis - - keyvalue - - database - maintainers: - - name: VMware, Inc. - url: https://github.com/bitnami/charts - name: redis - sources: - - https://github.com/bitnami/charts/tree/main/bitnami/redis - urls: - - https://berriai.github.io/litellm/charts/redis-18.19.1.tgz - version: 18.19.1 -generated: "2024-08-19T23:58:25.322532+08:00" diff --git a/litellm-js/proxy/.npmrc b/litellm-js/proxy/.npmrc deleted file mode 100644 index 7999681cc35..00000000000 --- a/litellm-js/proxy/.npmrc +++ /dev/null @@ -1,5 +0,0 @@ -# Supply-chain hardening -# Packages needing lifecycle scripts: npm rebuild -ignore-scripts=true -# Protects local npm install only — npm ci (used in CI) ignores this -min-release-age=3 diff --git a/litellm-js/proxy/README.md b/litellm-js/proxy/README.md deleted file mode 100644 index cc58e962d8f..00000000000 --- a/litellm-js/proxy/README.md +++ /dev/null @@ -1,8 +0,0 @@ -``` -npm install -npm run dev -``` - -``` -npm run deploy -``` diff --git a/litellm-js/proxy/package-lock.json b/litellm-js/proxy/package-lock.json deleted file mode 100644 index 0d09fa1a6c4..00000000000 --- a/litellm-js/proxy/package-lock.json +++ /dev/null @@ -1,2054 +0,0 @@ -{ - "name": "proxy", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "hono": "4.12.16", - "openai": "4.29.2" - }, - "devDependencies": { - "@cloudflare/workers-types": "4.20260501.1", - "wrangler": "4.87.0" - } - }, - "node_modules/@cloudflare/kv-asset-handler": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", - "integrity": "sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==", - "dev": true, - "license": "MIT OR Apache-2.0", - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@cloudflare/unenv-preset": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.16.1.tgz", - "integrity": "sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==", - "dev": true, - "license": "MIT OR Apache-2.0", - "peerDependencies": { - "unenv": "2.0.0-rc.24", - "workerd": ">1.20260305.0 <2.0.0-0" - }, - "peerDependenciesMeta": { - "workerd": { - "optional": true - } - } - }, - "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260430.1.tgz", - "integrity": "sha512-ADohZUHf7NBvPp2PdZig2Opxx+hDkk3ve7jrTne3JRx9kDSB73zc4LzcEeEN8LKkbAcqZmvfRJfpChSlusu0lA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260430.1.tgz", - "integrity": "sha512-/DoYC/1wHs+YRZzzqSQg1/EHB4hiv1yV5U8FnmapRRIzVaPtnt+ApeOXeMrIdKidgKOI8TqQzgBU8xbIM7Cl4Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260430.1.tgz", - "integrity": "sha512-koJhBWvEVZPKCVFtMLp2iMHlYr+lFCF47wGbnlKdHVlemV0zTxJEyHI8aLlrhPLhBmOmYLp46rXw09/qJkRIhQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260430.1.tgz", - "integrity": "sha512-hMdapNAzNQZDXGGkg4Slydc3fRJP5FUZLJVVcZCW/+imhhJro9Z1rv5n/wfR+txKoSWhTYR8eOp8Pyi2bzLzlw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260430.1.tgz", - "integrity": "sha512-jS3ffixjb5USOwz4frw4WzCz0HrjVxkgyU3WiYb06N7hBAfN6eOrveAJ4QRef0+suK4V1vQFoB1oKdRBsXe9Dw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workers-types": { - "version": "4.20260501.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260501.1.tgz", - "integrity": "sha512-B/VX2w3my/sCqxKyWOX7SxUpFC1uD8Gh7I2zbI1d3zA8p7Tx03AFsnuEx8lYLmcd8yONAA93YsAZb1wAaLK83w==", - "dev": true, - "license": "MIT OR Apache-2.0" - }, - "node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", - "integrity": "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.3.tgz", - "integrity": "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz", - "integrity": "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.3.tgz", - "integrity": "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz", - "integrity": "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz", - "integrity": "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz", - "integrity": "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz", - "integrity": "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz", - "integrity": "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz", - "integrity": "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz", - "integrity": "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz", - "integrity": "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz", - "integrity": "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz", - "integrity": "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz", - "integrity": "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz", - "integrity": "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.3.tgz", - "integrity": "sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz", - "integrity": "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz", - "integrity": "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz", - "integrity": "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz", - "integrity": "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz", - "integrity": "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz", - "integrity": "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz", - "integrity": "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz", - "integrity": "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz", - "integrity": "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/colour": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", - "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/sharp-darwin-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", - "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-darwin-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", - "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", - "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", - "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", - "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", - "cpu": [ - "arm" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", - "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", - "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", - "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", - "cpu": [ - "riscv64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", - "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", - "cpu": [ - "s390x" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", - "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", - "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", - "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-linux-arm": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", - "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", - "cpu": [ - "arm" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", - "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-ppc64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", - "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-riscv64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", - "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", - "cpu": [ - "riscv64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-s390x": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", - "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", - "cpu": [ - "s390x" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", - "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", - "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", - "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-wasm32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", - "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", - "optional": true, - "dependencies": { - "@emnapi/runtime": "^1.7.0" - }, - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", - "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-ia32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", - "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", - "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", - "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - }, - "node_modules/@poppinss/colors": { - "version": "4.1.6", - "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", - "integrity": "sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==", - "dev": true, - "license": "MIT", - "dependencies": { - "kleur": "^4.1.5" - } - }, - "node_modules/@poppinss/dumper": { - "version": "0.6.5", - "resolved": "https://registry.npmjs.org/@poppinss/dumper/-/dumper-0.6.5.tgz", - "integrity": "sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@poppinss/colors": "^4.1.5", - "@sindresorhus/is": "^7.0.2", - "supports-color": "^10.0.0" - } - }, - "node_modules/@poppinss/exception": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/@poppinss/exception/-/exception-1.2.3.tgz", - "integrity": "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@sindresorhus/is": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", - "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sindresorhus/is?sponsor=1" - } - }, - "node_modules/@speed-highlight/core": { - "version": "1.2.15", - "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.15.tgz", - "integrity": "sha512-BMq1K3DsElxDWawkX6eLg9+CKJrTVGCBAWVuHXVUV2u0s2711qiChLSId6ikYPfxhdYocLNt3wWwSvDiTvFabw==", - "dev": true, - "license": "CC0-1.0" - }, - "node_modules/@types/node": { - "version": "18.19.130", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", - "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", - "license": "MIT", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/@types/node-fetch": { - "version": "2.6.13", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", - "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.4" - } - }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "license": "MIT", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "license": "MIT", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, - "node_modules/base-64": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", - "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==" - }, - "node_modules/blake3-wasm": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", - "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", - "dev": true, - "license": "MIT" - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/charenc": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", - "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", - "license": "BSD-3-Clause", - "engines": { - "node": "*" - } - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/cookie": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", - "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/crypt": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", - "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", - "license": "BSD-3-Clause", - "engines": { - "node": "*" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, - "node_modules/digest-fetch": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz", - "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==", - "license": "ISC", - "dependencies": { - "base-64": "^0.1.0", - "md5": "^2.3.0" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/error-stack-parser-es": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", - "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/antfu" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/esbuild": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", - "integrity": "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.3", - "@esbuild/android-arm": "0.27.3", - "@esbuild/android-arm64": "0.27.3", - "@esbuild/android-x64": "0.27.3", - "@esbuild/darwin-arm64": "0.27.3", - "@esbuild/darwin-x64": "0.27.3", - "@esbuild/freebsd-arm64": "0.27.3", - "@esbuild/freebsd-x64": "0.27.3", - "@esbuild/linux-arm": "0.27.3", - "@esbuild/linux-arm64": "0.27.3", - "@esbuild/linux-ia32": "0.27.3", - "@esbuild/linux-loong64": "0.27.3", - "@esbuild/linux-mips64el": "0.27.3", - "@esbuild/linux-ppc64": "0.27.3", - "@esbuild/linux-riscv64": "0.27.3", - "@esbuild/linux-s390x": "0.27.3", - "@esbuild/linux-x64": "0.27.3", - "@esbuild/netbsd-arm64": "0.27.3", - "@esbuild/netbsd-x64": "0.27.3", - "@esbuild/openbsd-arm64": "0.27.3", - "@esbuild/openbsd-x64": "0.27.3", - "@esbuild/openharmony-arm64": "0.27.3", - "@esbuild/sunos-x64": "0.27.3", - "@esbuild/win32-arm64": "0.27.3", - "@esbuild/win32-ia32": "0.27.3", - "@esbuild/win32-x64": "0.27.3" - } - }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/form-data": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", - "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", - "license": "MIT" - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "license": "MIT", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, - "node_modules/formdata-node/node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", - "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/hono": { - "version": "4.12.16", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz", - "integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==", - "license": "MIT", - "engines": { - "node": ">=16.9.0" - } - }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.0.0" - } - }, - "node_modules/is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "license": "MIT" - }, - "node_modules/kleur": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", - "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/md5": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", - "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", - "license": "BSD-3-Clause", - "dependencies": { - "charenc": "0.0.2", - "crypt": "0.0.2", - "is-buffer": "~1.1.6" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/miniflare": { - "version": "4.20260430.0", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260430.0.tgz", - "integrity": "sha512-MWvMm3Siho9Yj7lbJZidLs8hbrRvIcOrif2mnsHQZdvoKfedpea+GaN8XJxbpRcq0B2WzNI1BB1ihdnqes3/ZA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@cspotcode/source-map-support": "0.8.1", - "sharp": "^0.34.5", - "undici": "7.24.8", - "workerd": "1.20260430.1", - "ws": "8.18.0", - "youch": "4.1.0-beta.10" - }, - "bin": { - "miniflare": "bootstrap.js" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/openai": { - "version": "4.29.2", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.29.2.tgz", - "integrity": "sha512-cPkT6zjEcE4qU5OW/SoDDuXEsdOLrXlAORhzmaguj5xZSPlgKvLhi27sFWhLKj07Y6WKNWxcwIbzm512FzTBNQ==", - "license": "Apache-2.0", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "digest-fetch": "^1.3.0", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7", - "web-streams-polyfill": "^3.2.1" - }, - "bin": { - "openai": "bin/cli" - } - }, - "node_modules/path-to-regexp": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", - "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/pathe": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", - "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", - "dev": true, - "license": "MIT" - }, - "node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/sharp": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", - "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", - "dev": true, - "hasInstallScript": true, - "license": "Apache-2.0", - "dependencies": { - "@img/colour": "^1.0.0", - "detect-libc": "^2.1.2", - "semver": "^7.7.3" - }, - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.34.5", - "@img/sharp-darwin-x64": "0.34.5", - "@img/sharp-libvips-darwin-arm64": "1.2.4", - "@img/sharp-libvips-darwin-x64": "1.2.4", - "@img/sharp-libvips-linux-arm": "1.2.4", - "@img/sharp-libvips-linux-arm64": "1.2.4", - "@img/sharp-libvips-linux-ppc64": "1.2.4", - "@img/sharp-libvips-linux-riscv64": "1.2.4", - "@img/sharp-libvips-linux-s390x": "1.2.4", - "@img/sharp-libvips-linux-x64": "1.2.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", - "@img/sharp-libvips-linuxmusl-x64": "1.2.4", - "@img/sharp-linux-arm": "0.34.5", - "@img/sharp-linux-arm64": "0.34.5", - "@img/sharp-linux-ppc64": "0.34.5", - "@img/sharp-linux-riscv64": "0.34.5", - "@img/sharp-linux-s390x": "0.34.5", - "@img/sharp-linux-x64": "0.34.5", - "@img/sharp-linuxmusl-arm64": "0.34.5", - "@img/sharp-linuxmusl-x64": "0.34.5", - "@img/sharp-wasm32": "0.34.5", - "@img/sharp-win32-arm64": "0.34.5", - "@img/sharp-win32-ia32": "0.34.5", - "@img/sharp-win32-x64": "0.34.5" - } - }, - "node_modules/supports-color": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", - "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD", - "optional": true - }, - "node_modules/undici": { - "version": "7.24.8", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.24.8.tgz", - "integrity": "sha512-6KQ/+QxK49Z/p3HO6E5ZCZWNnCasyZLa5ExaVYyvPxUwKtbCPMKELJOqh7EqOle0t9cH/7d2TaaTRRa6Nhs4YQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.18.1" - } - }, - "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "license": "MIT" - }, - "node_modules/unenv": { - "version": "2.0.0-rc.24", - "resolved": "https://registry.npmjs.org/unenv/-/unenv-2.0.0-rc.24.tgz", - "integrity": "sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "pathe": "^2.0.3" - } - }, - "node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/workerd": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260430.1.tgz", - "integrity": "sha512-KEgIWyiw3Jmn+DCd/L3ePo5fmiiYb/UcwKvDWPf/nLLOiwShDFzDSsegU5NY/JcwgvO/QsLHVi2FYrbkcXNY5Q==", - "dev": true, - "hasInstallScript": true, - "license": "Apache-2.0", - "bin": { - "workerd": "bin/workerd" - }, - "engines": { - "node": ">=16" - }, - "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20260430.1", - "@cloudflare/workerd-darwin-arm64": "1.20260430.1", - "@cloudflare/workerd-linux-64": "1.20260430.1", - "@cloudflare/workerd-linux-arm64": "1.20260430.1", - "@cloudflare/workerd-windows-64": "1.20260430.1" - } - }, - "node_modules/wrangler": { - "version": "4.87.0", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.87.0.tgz", - "integrity": "sha512-lfhfKwLfQlowwgV0xhlYgE9fU3n0I30d4ccGY/rTCEm/n42Mjvlr0Ng3ZPNqlsrsKBcDR531V7dsPkgELvrk/Q==", - "dev": true, - "license": "MIT OR Apache-2.0", - "dependencies": { - "@cloudflare/kv-asset-handler": "0.5.0", - "@cloudflare/unenv-preset": "2.16.1", - "blake3-wasm": "2.1.5", - "esbuild": "0.27.3", - "miniflare": "4.20260430.0", - "path-to-regexp": "6.3.0", - "unenv": "2.0.0-rc.24", - "workerd": "1.20260430.1" - }, - "bin": { - "wrangler": "bin/wrangler.js", - "wrangler2": "bin/wrangler.js" - }, - "engines": { - "node": ">=22.0.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - }, - "peerDependencies": { - "@cloudflare/workers-types": "^4.20260430.1" - }, - "peerDependenciesMeta": { - "@cloudflare/workers-types": { - "optional": true - } - } - }, - "node_modules/ws": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", - "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/youch": { - "version": "4.1.0-beta.10", - "resolved": "https://registry.npmjs.org/youch/-/youch-4.1.0-beta.10.tgz", - "integrity": "sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@poppinss/colors": "^4.1.5", - "@poppinss/dumper": "^0.6.4", - "@speed-highlight/core": "^1.2.7", - "cookie": "^1.0.2", - "youch-core": "^0.3.3" - } - }, - "node_modules/youch-core": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/youch-core/-/youch-core-0.3.3.tgz", - "integrity": "sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@poppinss/exception": "^1.2.2", - "error-stack-parser-es": "^1.0.5" - } - } - } -} diff --git a/litellm-js/proxy/package.json b/litellm-js/proxy/package.json deleted file mode 100644 index 9fd94cd882f..00000000000 --- a/litellm-js/proxy/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "scripts": { - "dev": "wrangler dev src/index.ts", - "deploy": "wrangler deploy --minify src/index.ts" - }, - "dependencies": { - "hono": "4.12.16", - "openai": "4.29.2" - }, - "devDependencies": { - "@cloudflare/workers-types": "4.20260501.1", - "wrangler": "4.87.0" - } -} diff --git a/litellm-js/proxy/src/index.ts b/litellm-js/proxy/src/index.ts deleted file mode 100644 index dc5dc9c689e..00000000000 --- a/litellm-js/proxy/src/index.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { Hono } from 'hono' -import { Context } from 'hono'; -import { bearerAuth } from 'hono/bearer-auth' -import OpenAI from "openai"; - -const openai = new OpenAI({ - apiKey: "sk-1234", - baseURL: "https://openai-endpoint.ishaanjaffer0324.workers.dev" -}); - -async function call_proxy() { - const completion = await openai.chat.completions.create({ - messages: [{ role: "system", content: "You are a helpful assistant." }], - model: "gpt-3.5-turbo", - }); - - return completion -} - -const app = new Hono() - -// Middleware for API Key Authentication -const apiKeyAuth = async (c: Context, next: Function) => { - const apiKey = c.req.header('Authorization'); - if (!apiKey || apiKey !== 'Bearer sk-1234') { - return c.text('Unauthorized', 401); - } - await next(); -}; - - -app.use('/*', apiKeyAuth) - - -app.get('/', (c) => { - return c.text('Hello Hono!') -}) - - - - -// Handler for chat completions -const chatCompletionHandler = async (c: Context) => { - // Assuming your logic for handling chat completion goes here - // For demonstration, just returning a simple JSON response - const response = await call_proxy() - return c.json(response); -}; - -// Register the above handler for different POST routes with the apiKeyAuth middleware -app.post('/v1/chat/completions', chatCompletionHandler); -app.post('/chat/completions', chatCompletionHandler); - -// Example showing how you might handle dynamic segments within the URL -// Here, using ':model*' to capture the rest of the path as a parameter 'model' -app.post('/openai/deployments/:model*/chat/completions', chatCompletionHandler); - - -export default app diff --git a/litellm-js/proxy/tsconfig.json b/litellm-js/proxy/tsconfig.json deleted file mode 100644 index 28fcfb58246..00000000000 --- a/litellm-js/proxy/tsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "Bundler", - "strict": true, - "lib": [ - "ESNext" - ], - "types": [ - "@cloudflare/workers-types" - ], - "jsx": "react-jsx", - "jsxImportSource": "hono/jsx", - "skipLibCheck": true - }, -} \ No newline at end of file diff --git a/litellm-js/proxy/wrangler.toml b/litellm-js/proxy/wrangler.toml deleted file mode 100644 index e7c323dff97..00000000000 --- a/litellm-js/proxy/wrangler.toml +++ /dev/null @@ -1,18 +0,0 @@ -name = "my-app" -compatibility_date = "2023-12-01" - -# [vars] -# MY_VAR = "my-variable" - -# [[kv_namespaces]] -# binding = "MY_KV_NAMESPACE" -# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - -# [[r2_buckets]] -# binding = "MY_BUCKET" -# bucket_name = "my-bucket" - -# [[d1_databases]] -# binding = "DB" -# database_name = "my-database" -# database_id = "" diff --git a/litellm-js/spend-logs/.npmrc b/litellm-js/spend-logs/.npmrc deleted file mode 100644 index 7999681cc35..00000000000 --- a/litellm-js/spend-logs/.npmrc +++ /dev/null @@ -1,5 +0,0 @@ -# Supply-chain hardening -# Packages needing lifecycle scripts: npm rebuild -ignore-scripts=true -# Protects local npm install only — npm ci (used in CI) ignores this -min-release-age=3 diff --git a/litellm-js/spend-logs/Dockerfile b/litellm-js/spend-logs/Dockerfile deleted file mode 100644 index 5040dc74bf6..00000000000 --- a/litellm-js/spend-logs/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Use the specific Node.js v20.11.0 image -FROM node:20.18.1-alpine3.20 - -# Set the working directory inside the container -WORKDIR /app - -# Copy package.json and package-lock.json to the working directory -COPY ./litellm-js/spend-logs/package*.json ./ - -# Install dependencies -RUN npm ci - -# Install Prisma globally -RUN npm install -g prisma - -# Copy the rest of the application code -COPY ./litellm-js/spend-logs . - -# Generate Prisma client -RUN npx prisma generate - -# Expose the port that the Node.js server will run on -EXPOSE 3000 - -# Command to run the Node.js app with npm run dev -CMD ["npm", "run", "dev"] diff --git a/litellm-js/spend-logs/README.md b/litellm-js/spend-logs/README.md deleted file mode 100644 index e12b31db70a..00000000000 --- a/litellm-js/spend-logs/README.md +++ /dev/null @@ -1,8 +0,0 @@ -``` -npm install -npm run dev -``` - -``` -open http://localhost:3000 -``` diff --git a/litellm-js/spend-logs/package-lock.json b/litellm-js/spend-logs/package-lock.json deleted file mode 100644 index e33079766c9..00000000000 --- a/litellm-js/spend-logs/package-lock.json +++ /dev/null @@ -1,597 +0,0 @@ -{ - "name": "spend-logs", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "@hono/node-server": "1.19.13", - "hono": "4.12.16" - }, - "devDependencies": { - "@types/node": "20.19.25", - "tsx": "4.20.6" - } - }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", - "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", - "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", - "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", - "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", - "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", - "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", - "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", - "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", - "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", - "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", - "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", - "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", - "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", - "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", - "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", - "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", - "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", - "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", - "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", - "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", - "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", - "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", - "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", - "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", - "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", - "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@hono/node-server": { - "version": "1.19.13", - "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.13.tgz", - "integrity": "sha512-TsQLe4i2gvoTtrHje625ngThGBySOgSK3Xo2XRYOdqGN1teR8+I7vchQC46uLJi8OF62YTYA3AhSpumtkhsaKQ==", - "license": "MIT", - "engines": { - "node": ">=18.14.1" - }, - "peerDependencies": { - "hono": "^4" - } - }, - "node_modules/@types/node": { - "version": "20.19.25", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.25.tgz", - "integrity": "sha512-ZsJzA5thDQMSQO788d7IocwwQbI8B5OPzmqNvpf3NY/+MHDAS759Wo0gd2WQeXYt5AAAQjzcrTVC6SKCuYgoCQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "node_modules/esbuild": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", - "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.12", - "@esbuild/android-arm": "0.25.12", - "@esbuild/android-arm64": "0.25.12", - "@esbuild/android-x64": "0.25.12", - "@esbuild/darwin-arm64": "0.25.12", - "@esbuild/darwin-x64": "0.25.12", - "@esbuild/freebsd-arm64": "0.25.12", - "@esbuild/freebsd-x64": "0.25.12", - "@esbuild/linux-arm": "0.25.12", - "@esbuild/linux-arm64": "0.25.12", - "@esbuild/linux-ia32": "0.25.12", - "@esbuild/linux-loong64": "0.25.12", - "@esbuild/linux-mips64el": "0.25.12", - "@esbuild/linux-ppc64": "0.25.12", - "@esbuild/linux-riscv64": "0.25.12", - "@esbuild/linux-s390x": "0.25.12", - "@esbuild/linux-x64": "0.25.12", - "@esbuild/netbsd-arm64": "0.25.12", - "@esbuild/netbsd-x64": "0.25.12", - "@esbuild/openbsd-arm64": "0.25.12", - "@esbuild/openbsd-x64": "0.25.12", - "@esbuild/openharmony-arm64": "0.25.12", - "@esbuild/sunos-x64": "0.25.12", - "@esbuild/win32-arm64": "0.25.12", - "@esbuild/win32-ia32": "0.25.12", - "@esbuild/win32-x64": "0.25.12" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/get-tsconfig": { - "version": "4.14.0", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz", - "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==", - "dev": true, - "license": "MIT", - "dependencies": { - "resolve-pkg-maps": "^1.0.0" - }, - "funding": { - "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" - } - }, - "node_modules/hono": { - "version": "4.12.16", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz", - "integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==", - "license": "MIT", - "engines": { - "node": ">=16.9.0" - } - }, - "node_modules/resolve-pkg-maps": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", - "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" - } - }, - "node_modules/tsx": { - "version": "4.20.6", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz", - "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==", - "dev": true, - "license": "MIT", - "dependencies": { - "esbuild": "~0.25.0", - "get-tsconfig": "^4.7.5" - }, - "bin": { - "tsx": "dist/cli.mjs" - }, - "engines": { - "node": ">=18.0.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - } - }, - "node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, - "license": "MIT" - } - } -} diff --git a/litellm-js/spend-logs/package.json b/litellm-js/spend-logs/package.json deleted file mode 100644 index 5a7a95c5de1..00000000000 --- a/litellm-js/spend-logs/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "scripts": { - "dev": "tsx watch src/index.ts" - }, - "dependencies": { - "@hono/node-server": "1.19.13", - "hono": "4.12.16" - }, - "devDependencies": { - "@types/node": "20.19.25", - "tsx": "4.20.6" - } -} diff --git a/litellm-js/spend-logs/schema.prisma b/litellm-js/spend-logs/schema.prisma deleted file mode 100644 index b0403f277aa..00000000000 --- a/litellm-js/spend-logs/schema.prisma +++ /dev/null @@ -1,29 +0,0 @@ -generator client { - provider = "prisma-client-js" -} - -datasource client { - provider = "postgresql" - url = env("DATABASE_URL") -} - -model LiteLLM_SpendLogs { - request_id String @id - call_type String - api_key String @default("") - spend Float @default(0.0) - total_tokens Int @default(0) - prompt_tokens Int @default(0) - completion_tokens Int @default(0) - startTime DateTime - endTime DateTime - model String @default("") - api_base String @default("") - user String @default("") - metadata Json @default("{}") - cache_hit String @default("") - cache_key String @default("") - request_tags Json @default("[]") - team_id String? - end_user String? -} \ No newline at end of file diff --git a/litellm-js/spend-logs/src/_types.ts b/litellm-js/spend-logs/src/_types.ts deleted file mode 100644 index 6a9b499171e..00000000000 --- a/litellm-js/spend-logs/src/_types.ts +++ /dev/null @@ -1,32 +0,0 @@ -export type LiteLLM_IncrementSpend = { - key_transactions: Array, // [{"key": spend},..] - user_transactions: Array, - team_transactions: Array, - spend_logs_transactions: Array -} - -export type LiteLLM_IncrementObject = { - key: string, - spend: number -} - -export type LiteLLM_SpendLogs = { - request_id: string; // @id means it's a unique identifier - call_type: string; - api_key: string; // @default("") means it defaults to an empty string if not provided - spend: number; // Float in Prisma corresponds to number in TypeScript - total_tokens: number; // Int in Prisma corresponds to number in TypeScript - prompt_tokens: number; - completion_tokens: number; - startTime: Date; // DateTime in Prisma corresponds to Date in TypeScript - endTime: Date; - model: string; // @default("") means it defaults to an empty string if not provided - api_base: string; - user: string; - metadata: any; // Json type in Prisma is represented by any in TypeScript; could also use a more specific type if the structure of JSON is known - cache_hit: string; - cache_key: string; - request_tags: any; // Similarly, this could be an array or a more specific type depending on the expected structure - team_id?: string | null; // ? indicates it's optional and can be undefined, but could also be null if not provided - end_user?: string | null; -}; \ No newline at end of file diff --git a/litellm-js/spend-logs/src/index.ts b/litellm-js/spend-logs/src/index.ts deleted file mode 100644 index 3581d95c830..00000000000 --- a/litellm-js/spend-logs/src/index.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { serve } from '@hono/node-server' -import { Hono } from 'hono' -import { PrismaClient } from '@prisma/client' -import {LiteLLM_SpendLogs, LiteLLM_IncrementSpend, LiteLLM_IncrementObject} from './_types' - -const app = new Hono() -const prisma = new PrismaClient() -// In-memory storage for logs -let spend_logs: LiteLLM_SpendLogs[] = []; -const key_logs: LiteLLM_IncrementObject[] = []; -const user_logs: LiteLLM_IncrementObject[] = []; -const transaction_logs: LiteLLM_IncrementObject[] = []; - - -app.get('/', (c) => { - return c.text('Hello Hono!') -}) - -const MIN_LOGS = 1; // Minimum number of logs needed to initiate a flush -const FLUSH_INTERVAL = 5000; // Time in ms to wait before trying to flush again -const BATCH_SIZE = 100; // Preferred size of each batch to write to the database -const MAX_LOGS_PER_INTERVAL = 1000; // Maximum number of logs to flush in a single interval - -const flushLogsToDb = async () => { - if (spend_logs.length >= MIN_LOGS) { - // Limit the logs to process in this interval to MAX_LOGS_PER_INTERVAL or less - const logsToProcess = spend_logs.slice(0, MAX_LOGS_PER_INTERVAL); - - for (let i = 0; i < logsToProcess.length; i += BATCH_SIZE) { - // Create subarray for current batch, ensuring it doesn't exceed the BATCH_SIZE - const batch = logsToProcess.slice(i, i + BATCH_SIZE); - - // Convert datetime strings to Date objects - const batchWithDates = batch.map(entry => ({ - ...entry, - startTime: new Date(entry.startTime), - endTime: new Date(entry.endTime), - // Repeat for any other DateTime fields you may have - })); - - await prisma.liteLLM_SpendLogs.createMany({ - data: batchWithDates, - }); - - console.log(`Flushed ${batch.length} logs to the DB.`); - } - - // Remove the processed logs from spend_logs - spend_logs = spend_logs.slice(logsToProcess.length); - - console.log(`${logsToProcess.length} logs processed. Remaining in queue: ${spend_logs.length}`); - } else { - // This will ensure it doesn't falsely claim "No logs to flush." when it's merely below the MIN_LOGS threshold. - if(spend_logs.length > 0) { - console.log(`Accumulating logs. Currently at ${spend_logs.length}, waiting for at least ${MIN_LOGS}.`); - } else { - console.log("No logs to flush."); - } - } -}; - -// Setup interval for attempting to flush the logs -setInterval(flushLogsToDb, FLUSH_INTERVAL); - -// Route to receive log messages -app.post('/spend/update', async (c) => { - const incomingLogs = await c.req.json(); - - spend_logs.push(...incomingLogs); - - console.log(`Received and stored ${incomingLogs.length} logs. Total logs in memory: ${spend_logs.length}`); - - return c.json({ message: `Successfully stored ${incomingLogs.length} logs` }); -}); - - - -const port = 3000 -console.log(`Server is running on port ${port}`) - -serve({ - fetch: app.fetch, - port -}) diff --git a/litellm-js/spend-logs/tsconfig.json b/litellm-js/spend-logs/tsconfig.json deleted file mode 100644 index 028c03b6a81..00000000000 --- a/litellm-js/spend-logs/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "Bundler", - "strict": true, - "types": [ - "node" - ], - "jsx": "react-jsx", - "jsxImportSource": "hono/jsx", - } -} \ No newline at end of file diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index f6f99eb62c8..0b30999aa21 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -2849,7 +2849,7 @@ def _can_object_call_model( object_type=object_type ), param="model", - code=status.HTTP_401_UNAUTHORIZED, + code=status.HTTP_403_FORBIDDEN, ) @@ -3082,7 +3082,7 @@ async def can_user_call_model( message=f"User not allowed to access model. No default model access, only team models allowed. Tried to access {model}", type=ProxyErrorTypes.key_model_access_denied, param="model", - code=status.HTTP_401_UNAUTHORIZED, + code=status.HTTP_403_FORBIDDEN, ) return _can_object_call_model( @@ -3625,7 +3625,7 @@ async def _check_team_member_model_access( message=f"Team member not allowed to access model. User={valid_token.user_id}, Team={team_object.team_id}, Model={model}. Allowed member models = {member_allowed_models}", type=ProxyErrorTypes.team_model_access_denied, param="model", - code=status.HTTP_401_UNAUTHORIZED, + code=status.HTTP_403_FORBIDDEN, ) diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index 5ded8136ef3..431db4254eb 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -123,7 +123,7 @@ class UserAPIKeyAuthExceptionHandler: message=e.message, type=ProxyErrorTypes.budget_exceeded, param=None, - code=400, + code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS), ) if isinstance(e, HTTPException): raise ProxyException( diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 9d3c06e641f..4778549befc 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1107,7 +1107,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 raise ProxyException( message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", type=ProxyErrorTypes.expired_key, - code=400, + code=status.HTTP_401_UNAUTHORIZED, param=abbreviate_api_key(api_key=api_key), ) valid_token = update_valid_token_with_end_user_params( @@ -1432,7 +1432,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 raise ProxyException( message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", type=ProxyErrorTypes.expired_key, - code=400, + code=status.HTTP_401_UNAUTHORIZED, param=abbreviate_api_key(api_key=api_key), ) @@ -2417,7 +2417,7 @@ async def _run_post_custom_auth_checks( raise ProxyException( message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", type=ProxyErrorTypes.expired_key, - code=400, + code=status.HTTP_401_UNAUTHORIZED, param=( abbreviate_api_key(api_key=valid_token.token) if valid_token.token diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 0928ce914da..71537cc62e6 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -2,7 +2,7 @@ import asyncio import json import time from datetime import datetime, timezone -from typing import Any, List, Literal, Optional, Union +from typing import Any, Callable, List, Literal, Optional, Union import litellm from litellm._logging import verbose_proxy_logger @@ -83,93 +83,139 @@ class ResetBudgetJob: "Failed to reset spend counter %s: %s", counter_key, e ) + @staticmethod + async def _invalidate_user_api_key_cache_entry(cache_key: str) -> None: + """Drop a stale management-cache entry so the next read fetches from DB. + + Some entity types (notably tags and end-users) are not handled by + SpendCounterReseed.from_db, so when a spend counter expires the + budget check falls back to ``cached_obj.spend``. If that cached + object lingers in ``user_api_key_cache`` past a budget reset, the + stale ``.spend`` keeps the entity blocked indefinitely. Deleting + the cache entry forces the next auth-time fetch to reload the + zeroed row from Postgres. + """ + try: + from litellm.proxy.proxy_server import user_api_key_cache + + await user_api_key_cache.async_delete_cache(key=cache_key) + except Exception as e: + verbose_proxy_logger.warning( + "Failed to invalidate user_api_key_cache entry %s: %s", + cache_key, + e, + ) + + async def _cascade_reset_spend_for_budget_link( + self, + budgets_to_reset: List[LiteLLM_BudgetTableFull], + table: Any, + counter_key_fn: Callable[[Any], str], + log_subject: str, + extra_where: Optional[dict] = None, + cache_key_fn: Optional[Callable[[Any], str]] = None, + ): + """ + Generic cascade: zero spend on rows whose budget_id is in the reset set. + + ``cache_key_fn`` is optional: when provided, after the DB update each + matching row's entry in ``user_api_key_cache`` is also dropped. This + is required for entities whose spend counter is read with the cached + object's ``.spend`` as fallback (tags, end-users) — otherwise the + stale cached object pins enforcement to the pre-reset spend until + its TTL expires. + """ + budget_ids = [b.budget_id for b in budgets_to_reset if b.budget_id is not None] + if not budget_ids: + return + + where: dict = {"budget_id": {"in": budget_ids}} + if extra_where: + where.update(extra_where) + + try: + rows = await table.find_many(where=where) + except Exception as e: + rows = [] + verbose_proxy_logger.warning( + "Failed to fetch %s for counter invalidation: %s", log_subject, e + ) + + update_result = await table.update_many(where=where, data={"spend": 0}) + + for row in rows: + await self._invalidate_spend_counter(counter_key_fn(row)) + if cache_key_fn is not None: + await self._invalidate_user_api_key_cache_entry(cache_key_fn(row)) + + return update_result + async def reset_budget_for_litellm_team_members( self, budgets_to_reset: List[LiteLLM_BudgetTableFull] ): """ Resets the budget for all LiteLLM Team Members if their budget has expired """ - budget_ids = [ - budget.budget_id - for budget in budgets_to_reset - if budget.budget_id is not None - ] - - try: - memberships = await self.prisma_client.db.litellm_teammembership.find_many( - where={"budget_id": {"in": budget_ids}} - ) - except Exception as e: - memberships = [] - verbose_proxy_logger.warning( - "Failed to fetch team memberships for counter invalidation: %s", e - ) - - update_result = await self.prisma_client.db.litellm_teammembership.update_many( - where={"budget_id": {"in": budget_ids}}, - data={ - "spend": 0, - }, + return await self._cascade_reset_spend_for_budget_link( + budgets_to_reset=budgets_to_reset, + table=self.prisma_client.db.litellm_teammembership, + counter_key_fn=lambda m: f"spend:team_member:{m.user_id}:{m.team_id}", + log_subject="team memberships", ) - for m in memberships: - await self._invalidate_spend_counter( - f"spend:team_member:{m.user_id}:{m.team_id}" - ) - - return update_result - async def reset_budget_for_keys_linked_to_budgets( self, budgets_to_reset: List[LiteLLM_BudgetTableFull] ): """ Resets the spend for keys linked to budget tiers that are being reset. - This handles keys that have budget_id but no budget_duration set on the key - itself. Keys with budget_id rely on their linked budget tier's reset schedule - rather than having their own budget_duration. - - Keys that have their own budget_duration are already handled by - reset_budget_for_litellm_keys() and are excluded here to avoid - double-resetting. + Excludes keys with their own budget_duration; those are reset by + reset_budget_for_litellm_keys() to avoid double-resetting. """ - budget_ids = [ - budget.budget_id - for budget in budgets_to_reset - if budget.budget_id is not None - ] - if not budget_ids: - return - - where_clause: dict = { - "budget_id": {"in": budget_ids}, - "budget_duration": None, # only keys without their own reset schedule - "spend": {"gt": 0}, # only reset keys that have accumulated spend - } - - try: - keys = await self.prisma_client.db.litellm_verificationtoken.find_many( - where=where_clause - ) - except Exception as e: - keys = [] - verbose_proxy_logger.warning( - "Failed to fetch keys for counter invalidation: %s", e - ) - - update_result = ( - await self.prisma_client.db.litellm_verificationtoken.update_many( - where=where_clause, - data={ - "spend": 0, - }, - ) + return await self._cascade_reset_spend_for_budget_link( + budgets_to_reset=budgets_to_reset, + table=self.prisma_client.db.litellm_verificationtoken, + counter_key_fn=lambda k: f"spend:key:{k.token}", + log_subject="keys", + extra_where={"budget_duration": None, "spend": {"gt": 0}}, ) - for k in keys: - await self._invalidate_spend_counter(f"spend:key:{k.token}") + async def reset_budget_for_orgs_linked_to_budgets( + self, budgets_to_reset: List[LiteLLM_BudgetTableFull] + ): + """ + Resets the spend for orgs linked to budget tiers that are being reset. + """ + return await self._cascade_reset_spend_for_budget_link( + budgets_to_reset=budgets_to_reset, + table=self.prisma_client.db.litellm_organizationtable, + counter_key_fn=lambda o: f"spend:org:{o.organization_id}", + log_subject="orgs", + extra_where={"spend": {"gt": 0}}, + ) - return update_result + async def reset_budget_for_tags_linked_to_budgets( + self, budgets_to_reset: List[LiteLLM_BudgetTableFull] + ): + """ + Resets the spend for tags linked to budget tiers that are being reset. + + Also drops each tag's ``user_api_key_cache`` entry so the next + ``_tag_max_budget_check`` reloads the zeroed row from the DB. + ``SpendCounterReseed.from_db`` intentionally returns ``None`` for + tags, so the budget check falls back to the cached + ``LiteLLM_TagTable.spend`` once the spend counter expires; without + this invalidation, that stale ``.spend`` keeps the tag over-budget + indefinitely. + """ + return await self._cascade_reset_spend_for_budget_link( + budgets_to_reset=budgets_to_reset, + table=self.prisma_client.db.litellm_tagtable, + counter_key_fn=lambda t: f"spend:tag:{t.tag_name}", + log_subject="tags", + extra_where={"spend": {"gt": 0}}, + cache_key_fn=lambda t: f"tag:{t.tag_name}", + ) async def reset_budget_for_litellm_budget_table(self): """ @@ -237,6 +283,14 @@ class ResetBudgetJob: budgets_to_reset=budgets_to_reset ) + await self.reset_budget_for_orgs_linked_to_budgets( + budgets_to_reset=budgets_to_reset + ) + + await self.reset_budget_for_tags_linked_to_budgets( + budgets_to_reset=budgets_to_reset + ) + if endusers_to_reset is not None and len(endusers_to_reset) > 0: for enduser in endusers_to_reset: try: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 02f9c9bef2f..493519f2328 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -211,6 +211,7 @@ from litellm import Router from litellm._logging import verbose_proxy_logger, verbose_router_logger from litellm.caching.caching import DualCache, RedisCache from litellm.caching.redis_cluster_cache import RedisClusterCache +from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.constants import ( _REALTIME_BODY_CACHE_SIZE, @@ -6750,27 +6751,64 @@ class ProxyStartupEvent: "budget_duration not set on Proxy. budget_duration is required to use max_budget." ) - # add proxy budget to db in the user table asyncio.create_task( - generate_key_helper_fn( # type: ignore - request_type="user", - table_name="user", - user_id=litellm_proxy_budget_name, - duration=None, - models=[], - aliases={}, - config={}, - spend=0, - max_budget=litellm.max_budget, - budget_duration=litellm.budget_duration, - query_type="update_data", - update_key_values={ - "max_budget": litellm.max_budget, - "budget_duration": litellm.budget_duration, - }, - ) + cls._upsert_proxy_budget_with_reset_at_backfill(litellm_proxy_budget_name) ) + @classmethod + async def _upsert_proxy_budget_with_reset_at_backfill( + cls, litellm_proxy_budget_name: str + ) -> None: + """ + Upsert the proxy admin user row with the configured max_budget / + budget_duration, then backfill budget_reset_at if currently NULL. + + The backfill uses `WHERE budget_reset_at IS NULL` so it only fires + when the row pre-existed without a reset schedule (e.g. row created + via a different path before the proxy budget was configured). On + subsequent restarts it no-ops, so an active reset window is never + slid forward. + """ + await generate_key_helper_fn( # type: ignore + request_type="user", + table_name="user", + user_id=litellm_proxy_budget_name, + duration=None, + models=[], + aliases={}, + config={}, + spend=0, + max_budget=litellm.max_budget, + budget_duration=litellm.budget_duration, + query_type="update_data", + update_key_values={ + "max_budget": litellm.max_budget, + "budget_duration": litellm.budget_duration, + }, + ) + + # Without this, the upsert leaves budget_reset_at=NULL on rows that + # took the UPDATE path, and reset_budget_for_litellm_users never + # matches them (NULL < now() is unknown in SQL) — so the proxy-wide + # spend cap blocks forever once it's hit. + if prisma_client is not None and litellm.budget_duration is not None: + try: + await prisma_client.db.litellm_usertable.update_many( + where={ + "user_id": litellm_proxy_budget_name, + "budget_reset_at": None, + }, + data={ + "budget_reset_at": get_budget_reset_time( + budget_duration=litellm.budget_duration + ) + }, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Failed to backfill budget_reset_at on proxy admin row: %s", e + ) + @classmethod async def _warm_global_spend_cache( cls, diff --git a/pyproject.toml b/pyproject.toml index d194d467913..5cd83148d37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "importlib-metadata>=8.0.0,<9.0", "tokenizers>=0.21.0,<1.0", "click>=8.0.0,<9.0", - "jinja2>=3.1.0,<4.0", + "jinja2>=3.1.6,<4.0", "aiohttp>=3.10,<4.0", "pydantic>=2.10.0,<3.0.0", "jsonschema>=4.0.0,<5.0", diff --git a/tests/litellm_utils_tests/test_proxy_budget_reset.py b/tests/litellm_utils_tests/test_proxy_budget_reset.py index a64c6c7aa36..6240bedd3e6 100644 --- a/tests/litellm_utils_tests/test_proxy_budget_reset.py +++ b/tests/litellm_utils_tests/test_proxy_budget_reset.py @@ -233,6 +233,12 @@ async def test_reset_budget_endusers_partial_failure(): prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( return_value={"count": 0} ) + # Mock db.litellm_organizationtable.update_many (used by reset_budget_for_orgs_linked_to_budgets) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 0} + ) + # Mock db.litellm_tagtable.update_many (used by reset_budget_for_tags_linked_to_budgets) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 0}) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() @@ -400,6 +406,12 @@ async def test_reset_budget_continues_other_categories_on_failure(): prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( return_value={"count": 0} ) + # Mock db.litellm_organizationtable.update_many (used by reset_budget_for_orgs_linked_to_budgets) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 0} + ) + # Mock db.litellm_tagtable.update_many (used by reset_budget_for_tags_linked_to_budgets) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 0}) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() @@ -884,6 +896,12 @@ async def test_service_logger_endusers_success(): prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( return_value={"count": 0} ) + # Mock db.litellm_organizationtable.update_many (used by reset_budget_for_orgs_linked_to_budgets) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 0} + ) + # Mock db.litellm_tagtable.update_many (used by reset_budget_for_tags_linked_to_budgets) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 0}) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() @@ -966,6 +984,12 @@ async def test_service_logger_endusers_failure(): prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( return_value={"count": 0} ) + # Mock db.litellm_organizationtable.update_many (used by reset_budget_for_orgs_linked_to_budgets) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 0} + ) + # Mock db.litellm_tagtable.update_many (used by reset_budget_for_tags_linked_to_budgets) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 0}) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() @@ -1060,6 +1084,10 @@ async def test_reset_budget_for_litellm_team_members_called(): prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( return_value={"count": 0} ) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 0} + ) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 0}) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() diff --git a/tests/otel_tests/test_e2e_budgeting.py b/tests/otel_tests/test_e2e_budgeting.py index 62fc8732ebd..f61befac4fb 100644 --- a/tests/otel_tests/test_e2e_budgeting.py +++ b/tests/otel_tests/test_e2e_budgeting.py @@ -25,8 +25,8 @@ async def make_calls_until_budget_exceeded(session, key: str, call_function, **k # Check error structure and values that should be consistent assert ( - error_dict["code"] == "400" - ), f"Expected error code 400, got: {error_dict['code']}" + error_dict["code"] == "429" + ), f"Expected error code 429, got: {error_dict['code']}" assert ( error_dict["type"] == "budget_exceeded" ), f"Expected error type budget_exceeded, got: {error_dict['type']}" diff --git a/tests/otel_tests/test_e2e_model_access.py b/tests/otel_tests/test_e2e_model_access.py index 7ea75a9d61d..87d85a19603 100644 --- a/tests/otel_tests/test_e2e_model_access.py +++ b/tests/otel_tests/test_e2e_model_access.py @@ -99,7 +99,7 @@ async def test_model_access_patterns(key_models, test_model, expect_success): # Assert error structure and values assert _error_body["type"] == "key_model_access_denied" assert _error_body["param"] == "model" - assert _error_body["code"] == "401" + assert _error_body["code"] == "403" assert "key not allowed to access model" in _error_body["message"] @@ -297,7 +297,7 @@ def _validate_model_access_exception( # Assert error structure and values assert _error_body["type"] == expected_type assert _error_body["param"] == "model" - assert _error_body["code"] == "401" + assert _error_body["code"] == "403" if expected_type == "key_model_access_denied": assert "key not allowed to access model" in _error_body["message"] elif expected_type == "team_model_access_denied": diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 8a854bcd6a8..26f04a4abcb 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -12,6 +12,7 @@ from datetime import datetime, timedelta import httpx import pytest +from fastapi import status import litellm from litellm.proxy._types import ( @@ -31,6 +32,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.auth_checks import ( ExperimentalUIJWTToken, + _can_object_call_model, _can_object_call_vector_stores, _check_end_user_budget, _check_team_member_budget, @@ -206,6 +208,52 @@ def test_get_key_object_from_ui_hash_key_invalid(): assert key_object is None +@pytest.mark.parametrize( + "object_type,expected_error_type", + [ + ("key", ProxyErrorTypes.key_model_access_denied), + ("team", ProxyErrorTypes.team_model_access_denied), + ("user", ProxyErrorTypes.user_model_access_denied), + ("org", ProxyErrorTypes.org_model_access_denied), + ("project", ProxyErrorTypes.project_model_access_denied), + ], +) +def test_can_object_call_model_denials_return_forbidden( + object_type, expected_error_type +): + with pytest.raises(ProxyException) as exc_info: + _can_object_call_model( + model="restricted-model", + llm_router=None, + models=["allowed-model"], + object_type=object_type, + ) + + assert exc_info.value.type == expected_error_type + assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN + + +@pytest.mark.asyncio +async def test_can_user_call_model_no_default_models_returns_forbidden(): + from litellm.proxy._types import SpecialModelNames + from litellm.proxy.auth.auth_checks import can_user_call_model + + user_object = LiteLLM_UserTable( + user_id="test-user", + models=[SpecialModelNames.no_default_models.value], + ) + + with pytest.raises(ProxyException) as exc_info: + await can_user_call_model( + model="restricted-model", + llm_router=None, + user_object=user_object, + ) + + assert exc_info.value.type == ProxyErrorTypes.key_model_access_denied + assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN + + @pytest.mark.asyncio async def test_get_key_object_should_reconnect_once_on_db_connection_error(): mock_prisma_client = MagicMock() @@ -1144,6 +1192,7 @@ async def test_check_team_member_model_access_denied_model(): proxy_logging_obj=MagicMock(), ) assert exc_info.value.type == ProxyErrorTypes.team_model_access_denied + assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index 2d1586f0b17..4ccde85dae2 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -140,6 +140,7 @@ async def test_handle_authentication_error_budget_exceeded(): ) assert exc_info.value.type == ProxyErrorTypes.budget_exceeded + assert int(exc_info.value.code) == status.HTTP_429_TOO_MANY_REQUESTS @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 95b3d746c66..50c5f43b218 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -1,6 +1,7 @@ import json import os import sys +from datetime import datetime, timedelta from types import SimpleNamespace from unittest.mock import ANY, AsyncMock, MagicMock, patch @@ -9,6 +10,7 @@ sys.path.insert( ) # Adds the parent directory to the system path import pytest +from fastapi import status import litellm import litellm.proxy.proxy_server @@ -178,6 +180,26 @@ async def test_custom_auth_does_not_enforce_key_model_access_by_default(): mock_can_key.assert_not_awaited() +@pytest.mark.asyncio +async def test_post_custom_auth_expired_key_returns_unauthorized(): + expired_token = UserAPIKeyAuth( + token="test_token", + expires=datetime.now() - timedelta(minutes=1), + ) + + with pytest.raises(ProxyException) as exc_info: + await _run_post_custom_auth_checks( + valid_token=expired_token, + request=MagicMock(), + request_data={}, + route="/v1/chat/completions", + parent_otel_span=None, + ) + + assert exc_info.value.type == ProxyErrorTypes.expired_key + assert int(exc_info.value.code) == status.HTTP_401_UNAUTHORIZED + + @pytest.mark.asyncio async def test_custom_auth_honors_key_level_model_access_restriction_allowed_with_opt_in(): valid_token = UserAPIKeyAuth(token="test_token", models=["gpt-4o-mini"]) @@ -934,6 +956,7 @@ async def test_proxy_admin_expired_key_from_cache(): assert ( exc_info.value.type == ProxyErrorTypes.expired_key ), f"Expected expired_key error type, got {exc_info.value.type}" + assert int(exc_info.value.code) == status.HTTP_401_UNAUTHORIZED assert "Expired Key" in str( exc_info.value.message ), f"Exception message should mention 'Expired Key', got: {exc_info.value.message}" diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py index 5c86f9057a1..8b0c76f836c 100644 --- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py +++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py @@ -39,6 +39,46 @@ class MockLiteLLMVerificationToken: return {"count": 1} +class MockLiteLLMOrganizationTable: + def __init__(self): + self.update_many_calls: List[Dict[str, Any]] = [] + self.find_many_calls: List[Dict[str, Any]] = [] + self._find_many_results: List[Any] = [] + + def set_find_many_results(self, results: List[Any]): + self._find_many_results = results + + async def find_many(self, where: Dict[str, Any]) -> List[Any]: + self.find_many_calls.append({"where": where}) + return self._find_many_results + + async def update_many( + self, where: Dict[str, Any], data: Dict[str, Any] + ) -> Dict[str, Any]: + self.update_many_calls.append({"where": where, "data": data}) + return {"count": 1} + + +class MockLiteLLMTagTable: + def __init__(self): + self.update_many_calls: List[Dict[str, Any]] = [] + self.find_many_calls: List[Dict[str, Any]] = [] + self._find_many_results: List[Any] = [] + + def set_find_many_results(self, results: List[Any]): + self._find_many_results = results + + async def find_many(self, where: Dict[str, Any]) -> List[Any]: + self.find_many_calls.append({"where": where}) + return self._find_many_results + + async def update_many( + self, where: Dict[str, Any], data: Dict[str, Any] + ) -> Dict[str, Any]: + self.update_many_calls.append({"where": where, "data": data}) + return {"count": 1} + + class MockLiteLLMEndUserTable: def __init__(self): self.find_many_calls: List[Dict[str, Any]] = [] @@ -57,6 +97,8 @@ class MockDB: self.litellm_teammembership = MockLiteLLMTeamMembership() self.litellm_verificationtoken = MockLiteLLMVerificationToken() self.litellm_endusertable = MockLiteLLMEndUserTable() + self.litellm_organizationtable = MockLiteLLMOrganizationTable() + self.litellm_tagtable = MockLiteLLMTagTable() class MockPrismaClient: @@ -459,6 +501,100 @@ def test_reset_budget_for_keys_linked_to_budgets_empty( assert len(calls) == 0 +def test_reset_budget_for_orgs_linked_to_budgets(reset_budget_job, mock_prisma_client): + """ + Test that when a budget tier is reset, orgs linked to that budget + (via budget_id) also get their spend reset. + """ + now = datetime.now(timezone.utc) + + test_budget = type( + "LiteLLM_BudgetTableFull", + (), + { + "max_budget": 100.0, + "budget_duration": "30d", + "budget_reset_at": now - timedelta(hours=1), + "budget_id": "30d-org-budget", + "created_at": now - timedelta(days=30), + }, + ) + + asyncio.run( + reset_budget_job.reset_budget_for_orgs_linked_to_budgets( + budgets_to_reset=[test_budget] + ) + ) + + calls = mock_prisma_client.db.litellm_organizationtable.update_many_calls + assert len(calls) == 1 + call = calls[0] + assert call["where"]["budget_id"] == {"in": ["30d-org-budget"]} + assert call["where"]["spend"] == {"gt": 0} + assert call["data"]["spend"] == 0 + + +def test_reset_budget_for_orgs_linked_to_budgets_empty( + reset_budget_job, mock_prisma_client +): + """ + Test that when there are no budgets to reset, no update is performed + on the organization table. + """ + asyncio.run( + reset_budget_job.reset_budget_for_orgs_linked_to_budgets(budgets_to_reset=[]) + ) + calls = mock_prisma_client.db.litellm_organizationtable.update_many_calls + assert len(calls) == 0 + + +def test_reset_budget_for_tags_linked_to_budgets(reset_budget_job, mock_prisma_client): + """ + Test that when a budget tier is reset, tags linked to that budget + (via budget_id) also get their spend reset. + """ + now = datetime.now(timezone.utc) + + test_budget = type( + "LiteLLM_BudgetTableFull", + (), + { + "max_budget": 50.0, + "budget_duration": "30d", + "budget_reset_at": now - timedelta(hours=1), + "budget_id": "30d-tag-budget", + "created_at": now - timedelta(days=30), + }, + ) + + asyncio.run( + reset_budget_job.reset_budget_for_tags_linked_to_budgets( + budgets_to_reset=[test_budget] + ) + ) + + calls = mock_prisma_client.db.litellm_tagtable.update_many_calls + assert len(calls) == 1 + call = calls[0] + assert call["where"]["budget_id"] == {"in": ["30d-tag-budget"]} + assert call["where"]["spend"] == {"gt": 0} + assert call["data"]["spend"] == 0 + + +def test_reset_budget_for_tags_linked_to_budgets_empty( + reset_budget_job, mock_prisma_client +): + """ + Test that when there are no budgets to reset, no update is performed + on the tag table. + """ + asyncio.run( + reset_budget_job.reset_budget_for_tags_linked_to_budgets(budgets_to_reset=[]) + ) + calls = mock_prisma_client.db.litellm_tagtable.update_many_calls + assert len(calls) == 0 + + @pytest.mark.parametrize( "budget_duration, expected_day, expected_month", [ @@ -618,6 +754,75 @@ def test_budget_table_reset_also_resets_linked_keys( assert calls[0]["data"]["spend"] == 0 +def test_budget_table_reset_also_resets_linked_orgs( + reset_budget_job, mock_prisma_client +): + """ + Integration-style test: when reset_budget_for_litellm_budget_table runs, + it should also reset spend for orgs linked to the expiring budget tiers + (in addition to end-users, team members, and keys). + """ + now = datetime.now(timezone.utc) + + test_budget = type( + "LiteLLM_BudgetTableFull", + (), + { + "max_budget": 100.0, + "budget_duration": "30d", + "budget_reset_at": now - timedelta(hours=1), + "budget_id": "30d-org-budget", + "created_at": now - timedelta(days=30), + }, + ) + + mock_prisma_client.data["budget"] = [test_budget] + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + calls = mock_prisma_client.db.litellm_organizationtable.update_many_calls + assert len(calls) == 1, ( + "Expected reset_budget_for_litellm_budget_table to also reset orgs " + f"linked to expiring budgets, but got {len(calls)} update_many calls" + ) + assert calls[0]["where"]["budget_id"] == {"in": ["30d-org-budget"]} + assert calls[0]["data"]["spend"] == 0 + + +def test_budget_table_reset_also_resets_linked_tags( + reset_budget_job, mock_prisma_client +): + """ + Integration-style test: when reset_budget_for_litellm_budget_table runs, + it should also reset spend for tags linked to the expiring budget tiers. + """ + now = datetime.now(timezone.utc) + + test_budget = type( + "LiteLLM_BudgetTableFull", + (), + { + "max_budget": 50.0, + "budget_duration": "30d", + "budget_reset_at": now - timedelta(hours=1), + "budget_id": "30d-tag-budget", + "created_at": now - timedelta(days=30), + }, + ) + + mock_prisma_client.data["budget"] = [test_budget] + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + calls = mock_prisma_client.db.litellm_tagtable.update_many_calls + assert len(calls) == 1, ( + "Expected reset_budget_for_litellm_budget_table to also reset tags " + f"linked to expiring budgets, but got {len(calls)} update_many calls" + ) + assert calls[0]["where"]["budget_id"] == {"in": ["30d-tag-budget"]} + assert calls[0]["data"]["spend"] == 0 + + def test_reset_budget_resets_endusers_with_null_budget_id( reset_budget_job, mock_prisma_client ): @@ -1057,16 +1262,26 @@ def test_reset_budget_windows_query_error_does_not_break_team_path(monkeypatch): def _make_counter_invalidation_job(monkeypatch): - """Stub spend_counter_cache so we can observe invalidation calls.""" + """Stub spend_counter_cache (and user_api_key_cache) so we can observe + invalidation calls. + + Both caches are looked up via ``from litellm.proxy.proxy_server import + `` inside the reset job, so we publish them on a fake module. + """ spend_counter_cache = MagicMock() spend_counter_cache.in_memory_cache.set_cache = MagicMock() spend_counter_cache.redis_cache = MagicMock() spend_counter_cache.redis_cache.async_set_cache = AsyncMock() + user_api_key_cache = MagicMock() + user_api_key_cache.async_delete_cache = AsyncMock() + fake_module = types.ModuleType("litellm.proxy.proxy_server") fake_module.spend_counter_cache = spend_counter_cache + fake_module.user_api_key_cache = user_api_key_cache monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", fake_module) + spend_counter_cache.user_api_key_cache = user_api_key_cache return spend_counter_cache @@ -1205,3 +1420,136 @@ def test_reset_budget_for_keys_linked_to_budgets_invalidates_redis_counter(monke counter_cache.in_memory_cache.set_cache.assert_any_call( key="spend:key:sk-linked", value=0.0, ttl=60 ) + + +def test_reset_budget_for_orgs_linked_to_budgets_invalidates_redis_counter(monkeypatch): + """Resetting orgs via budget tier must clear each linked org's counter.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_org = type("Org", (), {"organization_id": "org-acme"}) + + prisma_client = MagicMock() + prisma_client.db.litellm_organizationtable.find_many = AsyncMock( + return_value=[linked_org] + ) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 1} + ) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_orgs_linked_to_budgets([expired_budget])) + + counter_cache.in_memory_cache.set_cache.assert_any_call( + key="spend:org:org-acme", value=0.0, ttl=60 + ) + counter_cache.redis_cache.async_set_cache.assert_any_await( + key="spend:org:org-acme", value=0.0, ttl=60 + ) + + +def test_reset_budget_for_tags_linked_to_budgets_invalidates_redis_counter(monkeypatch): + """Resetting tags via budget tier must clear each linked tag's counter.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_tag = type("Tag", (), {"tag_name": "tenant-42"}) + + prisma_client = MagicMock() + prisma_client.db.litellm_tagtable.find_many = AsyncMock(return_value=[linked_tag]) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 1}) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_tags_linked_to_budgets([expired_budget])) + + counter_cache.in_memory_cache.set_cache.assert_any_call( + key="spend:tag:tenant-42", value=0.0, ttl=60 + ) + counter_cache.redis_cache.async_set_cache.assert_any_await( + key="spend:tag:tenant-42", value=0.0, ttl=60 + ) + + +def test_reset_budget_for_tags_linked_to_budgets_invalidates_management_cache( + monkeypatch, +): + """Regression guard for the bug where tag spend stayed frozen across cycles. + + ``SpendCounterReseed.from_db`` returns ``None`` for ``spend:tag:*`` keys, + so once the spend counter expires the tag budget check falls back to the + cached ``LiteLLM_TagTable.spend``. If we don't drop the management cache + entry on reset, that cached object lingers (TTL 60s) with the pre-reset + spend, and ``_tag_max_budget_check`` keeps returning HTTP 400 even though + the DB row has been zeroed. + """ + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_tag = type("Tag", (), {"tag_name": "tenant-42"}) + + prisma_client = MagicMock() + prisma_client.db.litellm_tagtable.find_many = AsyncMock(return_value=[linked_tag]) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 1}) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_tags_linked_to_budgets([expired_budget])) + + counter_cache.user_api_key_cache.async_delete_cache.assert_any_await( + key="tag:tenant-42" + ) + + +def test_reset_budget_for_tags_linked_to_budgets_invalidates_each_tag_management_cache( + monkeypatch, +): + """When multiple tags share the expired budget tier, every one of them + has its ``user_api_key_cache`` entry dropped — not just the first.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_tags = [ + type("Tag", (), {"tag_name": "tenant-a"}), + type("Tag", (), {"tag_name": "tenant-b"}), + type("Tag", (), {"tag_name": "tenant-c"}), + ] + + prisma_client = MagicMock() + prisma_client.db.litellm_tagtable.find_many = AsyncMock(return_value=linked_tags) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 3}) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_tags_linked_to_budgets([expired_budget])) + + deleted_keys = { + call.kwargs.get("key") + for call in counter_cache.user_api_key_cache.async_delete_cache.await_args_list + } + assert deleted_keys == {"tag:tenant-a", "tag:tenant-b", "tag:tenant-c"} + + +def test_reset_budget_for_keys_linked_to_budgets_does_not_touch_management_cache( + monkeypatch, +): + """Cache invalidation is opt-in: keys / orgs / team-members rely on + ``SpendCounterReseed.from_db`` (which DOES handle their counter keys), + so the cache_key_fn hook is intentionally not wired for them. This test + locks in that no-op so a future refactor doesn't accidentally start + clobbering the key cache (which would cost an extra DB round-trip per + reset cycle without fixing anything).""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_key = type("Key", (), {"token": "sk-linked"}) + + prisma_client = MagicMock() + prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[linked_key] + ) + prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( + return_value={"count": 1} + ) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_keys_linked_to_budgets([expired_budget])) + + counter_cache.user_api_key_cache.async_delete_cache.assert_not_awaited() diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 6718f52cbf1..859594f7a0b 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -1728,6 +1728,67 @@ async def test_add_proxy_budget_to_db_only_creates_user_no_keys(): assert call_args.kwargs["query_type"] == "update_data" +@pytest.mark.asyncio +async def test_add_proxy_budget_to_db_backfills_budget_reset_at(): + """ + Test that _upsert_proxy_budget_with_reset_at_backfill issues a conditional + update_many with `WHERE budget_reset_at IS NULL` to backfill the column on + rows that pre-existed without a reset schedule. Without this, the proxy + admin row stays at NULL and reset_budget_for_litellm_users never matches + it (NULL < now() is unknown in SQL), so the global proxy budget never + resets. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + import litellm + from litellm.proxy.proxy_server import ProxyStartupEvent + + litellm.budget_duration = "30d" + litellm.max_budget = 100.0 + litellm_proxy_budget_name = "litellm-proxy-budget" + + mock_prisma = MagicMock() + mock_prisma.db.litellm_usertable.update_many = AsyncMock(return_value={"count": 1}) + + mock_generate_key_helper = AsyncMock( + return_value={ + "user_id": litellm_proxy_budget_name, + "max_budget": 100.0, + "budget_duration": "30d", + "spend": 0, + "models": [], + } + ) + + with ( + patch( + "litellm.proxy.proxy_server.generate_key_helper_fn", + mock_generate_key_helper, + ), + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + ): + await ProxyStartupEvent._upsert_proxy_budget_with_reset_at_backfill( + litellm_proxy_budget_name + ) + + # Upsert ran with the configured budget + mock_generate_key_helper.assert_called_once() + + # Backfill update_many ran with the conditional WHERE + mock_prisma.db.litellm_usertable.update_many.assert_called_once() + backfill_call = mock_prisma.db.litellm_usertable.update_many.call_args + assert backfill_call.kwargs["where"]["user_id"] == litellm_proxy_budget_name + assert backfill_call.kwargs["where"]["budget_reset_at"] is None + + # The backfilled value must be a real future datetime — anything else and + # reset_budget_for_litellm_users would still skip the row. + from datetime import datetime, timezone + + backfilled_reset_at = backfill_call.kwargs["data"]["budget_reset_at"] + assert isinstance(backfilled_reset_at, datetime) + assert backfilled_reset_at > datetime.now(timezone.utc) + + @pytest.mark.asyncio async def test_custom_ui_sso_sign_in_handler_config_loading(): """ diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py index 024d05e1037..8a3f9361ba1 100644 --- a/tests/test_openai_endpoints.py +++ b/tests/test_openai_endpoints.py @@ -303,7 +303,7 @@ async def test_chat_completion(): api_key=key_gen["key"], api_version="2024-02-15-preview", ) - with pytest.raises(openai.AuthenticationError) as e: + with pytest.raises(openai.PermissionDeniedError) as e: response = await azure_client.chat.completions.create( model="gpt-4", messages=[{"role": "user", "content": "Hello!"}], diff --git a/tests/test_users.py b/tests/test_users.py index 05253a19aa5..57fbb0483e4 100644 --- a/tests/test_users.py +++ b/tests/test_users.py @@ -302,14 +302,14 @@ async def test_user_model_access(): model="good-model", ) - with pytest.raises(openai.AuthenticationError): + with pytest.raises(openai.PermissionDeniedError): await chat_completion( session=session, key=key, model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", ) - with pytest.raises(openai.AuthenticationError): + with pytest.raises(openai.PermissionDeniedError): await chat_completion( session=session, key=key, diff --git a/uv.lock b/uv.lock index f8d78fe8794..ab9aba1e38f 100644 --- a/uv.lock +++ b/uv.lock @@ -3405,7 +3405,7 @@ requires-dist = [ { name = "gunicorn", marker = "extra == 'proxy'", specifier = "==23.0.0" }, { name = "httpx", specifier = ">=0.28.0,<1.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, - { name = "jinja2", specifier = ">=3.1.0,<4.0" }, + { name = "jinja2", specifier = ">=3.1.6,<4.0" }, { name = "jsonschema", specifier = ">=4.0.0,<5.0" }, { name = "langfuse", marker = "extra == 'proxy-runtime'", specifier = "==2.59.7" }, { name = "litellm-enterprise", marker = "extra == 'proxy'", editable = "enterprise" }, From 0af33fbe7004d8d13aaa912373bba2d20e62042b Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 9 May 2026 19:01:58 -0700 Subject: [PATCH 55/85] fix(ui): omit allowed_routes from key edit save when unchanged (#27553) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ui): omit allowed_routes from key edit save when unchanged When a team admin opens Edit Settings on a key with key_type=AI APIs and saves without changing anything, the UI re-sends the existing allowed_routes value, which the backend's _check_allowed_routes_caller_permission gate rejects for non-proxy-admins (LIT-2681). Strip allowed_routes from the patch in handleSubmit when it deep-equals the original keyData.allowed_routes. The backend treats absence as "leave alone," so no-op saves now succeed for non-admins. Admins explicitly editing the field still send the new value. * fix(ui): order-insensitive allowed_routes diff + cover null-original case Address Greptile review: - Switch the "is allowed_routes unchanged" check to a Set-based comparison so a server-side reorder of the array doesn't register as a user edit and re-trigger LIT-2681. - Add two regression tests: (1) keyData.allowed_routes is null and the form is untouched — patch should strip the field; (2) server returned routes in a different order than the user originally entered — patch should still recognize the value as unchanged. * chore(ui): strip ticket refs and tighten comments in key edit fix - Remove internal-tracker references from in-code comments - Tighten the WHY comment in handleSubmit to two lines - Drop redundant test-block comments — test names already describe the case * fix(ui): annotate Set generic in allowed_routes diff to fix tsc --- .../templates/key_edit_view.test.tsx | 162 ++++++++++++++---- .../components/templates/key_edit_view.tsx | 82 +++++---- 2 files changed, 180 insertions(+), 64 deletions(-) diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx index 2e4d0d97e4c..1886075a9d9 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx @@ -158,8 +158,8 @@ describe("KeyEditView", () => { const { getByText } = renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -176,8 +176,8 @@ describe("KeyEditView", () => { const { getByText } = renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -194,8 +194,8 @@ describe("KeyEditView", () => { const { getByLabelText } = renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -219,7 +219,7 @@ describe("KeyEditView", () => { { }} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -241,8 +241,8 @@ describe("KeyEditView", () => { renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -259,8 +259,8 @@ describe("KeyEditView", () => { renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -277,8 +277,8 @@ describe("KeyEditView", () => { renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -295,8 +295,8 @@ describe("KeyEditView", () => { renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -314,7 +314,7 @@ describe("KeyEditView", () => { renderWithProviders( { }} + onCancel={() => {}} onSubmit={onSubmitMock} accessToken={"test-token"} userID={"test-user"} @@ -344,8 +344,8 @@ describe("KeyEditView", () => { renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -367,8 +367,8 @@ describe("KeyEditView", () => { renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -385,8 +385,8 @@ describe("KeyEditView", () => { renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={"test-token"} userID={""} userRole={""} @@ -404,7 +404,7 @@ describe("KeyEditView", () => { renderWithProviders( { }} + onCancel={() => {}} onSubmit={onSubmitMock} accessToken={"test-token"} userID={"test-user"} @@ -434,10 +434,14 @@ describe("KeyEditView", () => { it("should handle empty allowed routes string on submit", async () => { const onSubmitMock = vi.fn().mockResolvedValue(undefined); + const keyDataWithRoutes = { + ...MOCK_KEY_DATA, + allowed_routes: ["llm_api_routes"], + }; renderWithProviders( { }} + keyData={keyDataWithRoutes} + onCancel={() => {}} onSubmit={onSubmitMock} accessToken={"test-token"} userID={"test-user"} @@ -463,6 +467,101 @@ describe("KeyEditView", () => { }); }); + it("should omit allowed_routes from submit when value is unchanged", async () => { + const onSubmitMock = vi.fn().mockResolvedValue(undefined); + const aiApisKeyData = { + ...MOCK_KEY_DATA, + allowed_routes: ["llm_api_routes"], + }; + renderWithProviders( + {}} + onSubmit={onSubmitMock} + accessToken={"test-token"} + userID={"test-user"} + userRole={"admin"} + premiumUser={false} + />, + ); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /save changes/i })).toBeInTheDocument(); + }); + + const submitButton = screen.getByRole("button", { name: /save changes/i }); + await userEvent.click(submitButton); + + await waitFor(() => { + expect(onSubmitMock).toHaveBeenCalled(); + const callArgs = onSubmitMock.mock.calls[0][0]; + expect("allowed_routes" in callArgs).toBe(false); + }); + }); + + it("should omit allowed_routes from submit when keyData.allowed_routes is null and form is untouched", async () => { + const onSubmitMock = vi.fn().mockResolvedValue(undefined); + const keyDataNullRoutes = { + ...MOCK_KEY_DATA, + allowed_routes: null as unknown as string[], + }; + renderWithProviders( + {}} + onSubmit={onSubmitMock} + accessToken={"test-token"} + userID={"test-user"} + userRole={"admin"} + premiumUser={false} + />, + ); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /save changes/i })).toBeInTheDocument(); + }); + + const submitButton = screen.getByRole("button", { name: /save changes/i }); + await userEvent.click(submitButton); + + await waitFor(() => { + expect(onSubmitMock).toHaveBeenCalled(); + const callArgs = onSubmitMock.mock.calls[0][0]; + expect("allowed_routes" in callArgs).toBe(false); + }); + }); + + it("should omit allowed_routes from submit when server returned routes in a different order", async () => { + const onSubmitMock = vi.fn().mockResolvedValue(undefined); + const keyDataReordered = { + ...MOCK_KEY_DATA, + allowed_routes: ["beta_routes", "alpha_routes"], + }; + renderWithProviders( + {}} + onSubmit={onSubmitMock} + accessToken={"test-token"} + userID={"test-user"} + userRole={"admin"} + premiumUser={false} + />, + ); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /save changes/i })).toBeInTheDocument(); + }); + + const submitButton = screen.getByRole("button", { name: /save changes/i }); + await userEvent.click(submitButton); + + await waitFor(() => { + expect(onSubmitMock).toHaveBeenCalled(); + const callArgs = onSubmitMock.mock.calls[0][0]; + expect("allowed_routes" in callArgs).toBe(false); + }); + }); it("should pass access_group_ids to onSubmit when saving key with access groups", async () => { const onSubmitMock = vi.fn().mockResolvedValue(undefined); @@ -554,7 +653,7 @@ describe("KeyEditView", () => { renderWithProviders( { }} + onCancel={() => {}} onSubmit={onSubmitMock} accessToken={"test-token"} userID={"test-user"} @@ -576,10 +675,13 @@ describe("KeyEditView", () => { }); // Wait for the cancel button to actually be disabled (state update may take a moment) - await waitFor(() => { - const cancelButton = screen.getByRole("button", { name: /cancel/i }); - expect(cancelButton).toBeDisabled(); - }, { timeout: 3000 }); + await waitFor( + () => { + const cancelButton = screen.getByRole("button", { name: /cancel/i }); + expect(cancelButton).toBeDisabled(); + }, + { timeout: 3000 }, + ); // Clean up: resolve the promise to allow the form to complete if (resolveSubmit) { diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index 9b38d930a50..d5e410029a7 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -78,7 +78,6 @@ const getKeyTypeFromRoutes = (allowedRoutes: string[] | null | undefined): strin return "default"; }; - export function KeyEditView({ keyData, onCancel, @@ -106,7 +105,7 @@ export function KeyEditView({ const [neverExpire, setNeverExpire] = useState(!keyData.expires); const [isKeySaving, setIsKeySaving] = useState(false); const [budgetLimits, setBudgetLimits] = useState( - Array.isArray(keyData.budget_limits) ? keyData.budget_limits : [] + Array.isArray(keyData.budget_limits) ? keyData.budget_limits : [], ); const { data: organizations, isLoading: isOrganizationsLoading } = useOrganizations(); const { data: projects } = useProjects(); @@ -116,9 +115,7 @@ export function KeyEditView({ const projectDisplay = (() => { if (!keyData.project_id) return null; const project = projects?.find((p) => p.project_id === keyData.project_id); - return project?.project_alias - ? `${project.project_alias} (${keyData.project_id})` - : keyData.project_id; + return project?.project_alias ? `${project.project_alias} (${keyData.project_id})` : keyData.project_id; })(); useEffect(() => { @@ -198,9 +195,10 @@ export function KeyEditView({ access_group_ids: keyData.access_group_ids || [], auto_rotate: keyData.auto_rotate || false, ...(keyData.rotation_interval && { rotation_interval: keyData.rotation_interval }), - allowed_routes: Array.isArray(keyData.allowed_routes) && keyData.allowed_routes.length > 0 - ? keyData.allowed_routes.join(", ") - : "", + allowed_routes: + Array.isArray(keyData.allowed_routes) && keyData.allowed_routes.length > 0 + ? keyData.allowed_routes.join(", ") + : "", }; useEffect(() => { @@ -226,9 +224,10 @@ export function KeyEditView({ access_group_ids: keyData.access_group_ids || [], auto_rotate: keyData.auto_rotate || false, ...(keyData.rotation_interval && { rotation_interval: keyData.rotation_interval }), - allowed_routes: Array.isArray(keyData.allowed_routes) && keyData.allowed_routes.length > 0 - ? keyData.allowed_routes.join(", ") - : "", + allowed_routes: + Array.isArray(keyData.allowed_routes) && keyData.allowed_routes.length > 0 + ? keyData.allowed_routes.join(", ") + : "", }); }, [keyData, form]); @@ -275,12 +274,25 @@ export function KeyEditView({ } // If it's already an array (shouldn't happen, but handle it), keep as is + // Backend rejects non-empty allowed_routes from non-admins, so re-sending + // an unchanged value 403s a team admin. Set compare tolerates reorder. + const originalRoutesSet = new Set(Array.isArray(keyData.allowed_routes) ? keyData.allowed_routes : []); + const submittedRoutesSet = new Set(Array.isArray(values.allowed_routes) ? values.allowed_routes : []); + const allowedRoutesUnchanged = + originalRoutesSet.size === submittedRoutesSet.size && + [...submittedRoutesSet].every((r) => originalRoutesSet.has(r)); + if (allowedRoutesUnchanged) { + delete values.allowed_routes; + } + if (neverExpire) { values.duration = null; } // Include multi-window budget limits (filter out incomplete entries) - const validWindows = budgetLimits.filter((w) => w.budget_duration && w.max_budget !== null && w.max_budget !== undefined); + const validWindows = budgetLimits.filter( + (w) => w.budget_duration && w.max_budget !== null && w.max_budget !== undefined, + ); values.budget_limits = validWindows.length > 0 ? validWindows : undefined; await onSubmit(values); @@ -305,9 +317,13 @@ export function KeyEditView({ {({ getFieldValue, setFieldValue }) => { const allowedRoutesValue = getFieldValue("allowed_routes") || ""; // Convert string to array for checking - const allowedRoutes = typeof allowedRoutesValue === "string" && allowedRoutesValue.trim() !== "" - ? allowedRoutesValue.split(",").map((r: string) => r.trim()).filter((r: string) => r.length > 0) - : []; + const allowedRoutes = + typeof allowedRoutesValue === "string" && allowedRoutesValue.trim() !== "" + ? allowedRoutesValue + .split(",") + .map((r: string) => r.trim()) + .filter((r: string) => r.length > 0) + : []; const isDisabled = allowedRoutes.includes("management_routes") || allowedRoutes.includes("info_routes"); const models = getFieldValue("models") || []; @@ -348,9 +364,13 @@ export function KeyEditView({ {({ getFieldValue, setFieldValue }) => { const allowedRoutesValue = getFieldValue("allowed_routes") || ""; // Convert string to array for getKeyTypeFromRoutes - const allowedRoutes = typeof allowedRoutesValue === "string" && allowedRoutesValue.trim() !== "" - ? allowedRoutesValue.split(",").map((r: string) => r.trim()).filter((r: string) => r.length > 0) - : []; + const allowedRoutes = + typeof allowedRoutesValue === "string" && allowedRoutesValue.trim() !== "" + ? allowedRoutesValue + .split(",") + .map((r: string) => r.trim()) + .filter((r: string) => r.length > 0) + : []; const keyTypeValue = getKeyTypeFromRoutes(allowedRoutes); return ( @@ -415,9 +435,7 @@ export function KeyEditView({ } name="allowed_routes" > - + @@ -442,10 +460,7 @@ export function KeyEditView({ } > - + @@ -579,7 +594,7 @@ export function KeyEditView({ !premiumUser ? "Premium feature - Upgrade to set allowed pass through routes by key" : Array.isArray(keyData.metadata?.allowed_passthrough_routes) && - keyData.metadata.allowed_passthrough_routes.length > 0 + keyData.metadata.allowed_passthrough_routes.length > 0 ? `Current: ${keyData.metadata.allowed_passthrough_routes.join(", ")}` : "Select or enter allowed pass through routes" } @@ -690,14 +705,13 @@ export function KeyEditView({ return team.team_alias?.toLowerCase().includes(input.toLowerCase()) ?? false; }} > - {(selectedOrganizationId - ? teams?.filter((t) => t.organization_id === selectedOrganizationId) - : teams - )?.map((team) => ( - - {`${team.team_alias} (${team.team_id})`} - - ))} + {(selectedOrganizationId ? teams?.filter((t) => t.organization_id === selectedOrganizationId) : teams)?.map( + (team) => ( + + {`${team.team_alias} (${team.team_id})`} + + ), + )} {enableProjectsUI && hasProject && ( From 99218c6fa0326deb0ff7c1a8ee84f00cd693c7c0 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 11 May 2026 09:49:47 +0530 Subject: [PATCH 56/85] Fix deprecated model test --- tests/llm_translation/test_openrouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/llm_translation/test_openrouter.py b/tests/llm_translation/test_openrouter.py index 631b0770e3d..8fbb8803d11 100644 --- a/tests/llm_translation/test_openrouter.py +++ b/tests/llm_translation/test_openrouter.py @@ -11,7 +11,7 @@ import litellm def test_completion_openrouter_reasoning_content(): litellm._turn_on_debug() resp = litellm.completion( - model="openrouter/anthropic/claude-3.7-sonnet", + model="openrouter/anthropic/claude-sonnet-4", messages=[{"role": "user", "content": "Hello world"}], reasoning={"effort": "high"}, ) From 157d81368f62dbbc95b91da10587d01ce283156e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 11 May 2026 10:42:19 +0530 Subject: [PATCH 57/85] fix(openai): route reasoningSummary on gpt-5.4+ chat without tools to Responses API - Extend responses_api_bridge_check when reasoning_effort + summary aliases (including nested extra_body) without tools - Merge summary into reasoning_effort for responses bridge; helpers in utils - Strip summary aliases in GPT-5 chat mapping when not bridged - Tests for bridge + merge behavior Co-authored-by: Cursor --- .../llms/openai/chat/gpt_5_transformation.py | 22 +++++++- litellm/main.py | 41 +++++++++++--- litellm/utils.py | 56 +++++++++++++++++++ tests/test_litellm/test_main.py | 45 +++++++++++++++ 4 files changed, 156 insertions(+), 8 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 4e34d10b187..eb7bb384a7e 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -3,7 +3,11 @@ from typing import Optional, Union import litellm -from litellm.utils import _is_explicitly_disabled_factory, _supports_factory +from litellm.utils import ( + _is_explicitly_disabled_factory, + _supports_factory, + strip_reasoning_summary_aliases_from_openai_completion_params, +) from .gpt_transformation import OpenAIGPTConfig @@ -191,6 +195,16 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if param not in non_supported_params ] + @staticmethod + def _strip_reasoning_summary_aliases_for_chat_completions( + non_default_params: dict, + optional_params: dict, + ) -> None: + """Remove Responses-style reasoning summary keys; invalid on Chat Completions.""" + strip_reasoning_summary_aliases_from_openai_completion_params( + non_default_params, optional_params + ) + def map_openai_params( self, non_default_params: dict, @@ -210,6 +224,12 @@ class OpenAIGPT5Config(OpenAIGPTConfig): drop_params=drop_params, ) + # AI SDK / Responses-style aliases; never valid on Chat Completions when not + # bridged to Responses API (see main.responses_api_bridge_check). + self._strip_reasoning_summary_aliases_for_chat_completions( + non_default_params, optional_params + ) + # Get raw reasoning_effort and effective effort level for all guards. # Use effective_effort (extracted string) for xhigh validation, "none" checks, and # tool/sampling guards — dict inputs like {"effort": "none", "summary": "detailed"} diff --git a/litellm/main.py b/litellm/main.py index 051a82fdd19..6a3e5db2057 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1,3 +1,5 @@ +# LiteLLM main module: public completion, embedding, streaming, and moderation entrypoints. +# # +-----------------------------------------------+ # | | # | Give Feedback / Get Help | @@ -59,7 +61,13 @@ import litellm from litellm import client # Other utils are imported directly to avoid circular imports -from litellm.utils import exception_type, get_litellm_params, get_optional_params +from litellm.utils import ( + exception_type, + get_litellm_params, + get_optional_params, + peek_reasoning_summary_aliases, + strip_reasoning_summary_aliases_from_optional_params, +) # Logging is imported lazily when needed to avoid loading litellm_logging at import time if TYPE_CHECKING: @@ -946,6 +954,7 @@ def responses_api_bridge_check( web_search_options: Optional[OpenAIWebSearchOptions] = None, tools: Optional[List[Any]] = None, reasoning_effort: Optional[Any] = None, + reasoning_summary: Optional[Any] = None, ) -> Tuple[dict, str]: model_info: Dict[str, Any] = {} @@ -982,13 +991,16 @@ def responses_api_bridge_check( mode = "responses" model_info["mode"] = mode - # OpenAI/Azure gpt-5.4+ chat-completions calls with both tools + reasoning_effort - # must be bridged to Responses API. + # OpenAI/Azure gpt-5.4+ chat-completions calls that need Responses-only fields + # (e.g. reasoning summary) must be bridged. SDKs send ``reasoningSummary`` / + # ``reasoning_summary`` alongside ``reasoning_effort``; Chat Completions rejects + # those keys, so route when tools+reasoning_effort (original case) or when a + # reasoning summary is requested without tools. if ( custom_llm_provider in ("openai", "azure") and OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) - and tools and reasoning_effort is not None + and (tools or reasoning_summary is not None) and model_info.get("mode") != "responses" ): model_info["mode"] = "responses" @@ -1634,8 +1646,10 @@ def completion( # type: ignore # noqa: PLR0915 ## RESPONSES API BRIDGE LOGIC ## - check if model has 'mode: responses' in litellm.model_cost map # Only run the second bridge check if the first one didn't already # detect responses mode (e.g. via the "responses/" prefix). The second - # check handles cases like gpt-5.4+ with tools+reasoning_effort that - # the first (early) check doesn't cover. + # check handles cases like gpt-5.4+ with tools+reasoning_effort or + # reasoningSummary/reasoning_summary without tools (AI SDK) that the first + # (early) check doesn't cover. + _reasoning_summary_for_bridge = peek_reasoning_summary_aliases(optional_params) if responses_api_model_info.get("mode") != "responses": responses_api_model_info, model = responses_api_bridge_check( model=model, @@ -1643,14 +1657,27 @@ def completion( # type: ignore # noqa: PLR0915 web_search_options=web_search_options, tools=tools, reasoning_effort=reasoning_effort, + reasoning_summary=_reasoning_summary_for_bridge, ) if responses_api_model_info.get("mode") == "responses": from litellm.completion_extras import responses_api_bridge + optional_params, rs_val = ( + strip_reasoning_summary_aliases_from_optional_params(optional_params) + ) + if isinstance(reasoning_effort, dict) and "summary" in reasoning_effort: - optional_params = dict(optional_params) optional_params["reasoning_effort"] = reasoning_effort + elif rs_val is not None: + eff = optional_params.get("reasoning_effort", reasoning_effort) + if isinstance(eff, dict): + optional_params["reasoning_effort"] = {**eff, "summary": rs_val} + elif eff is not None: + optional_params["reasoning_effort"] = { + "effort": eff, + "summary": rs_val, + } return responses_api_bridge.completion( model=model, diff --git a/litellm/utils.py b/litellm/utils.py index 019fbc2add8..d473446b180 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1,3 +1,5 @@ +"""Utility helpers for LiteLLM core request handling and provider support.""" + # from __future__ import annotations must be the first non-comment statement from __future__ import annotations @@ -9490,6 +9492,60 @@ def get_non_default_completion_params(kwargs: dict) -> dict: return non_default_params +def peek_reasoning_summary_aliases(optional_params: dict) -> Optional[Any]: + """Read AI-SDK-style reasoning summary from optional_params or nested extra_body.""" + rs = optional_params.get("reasoningSummary") or optional_params.get( + "reasoning_summary" + ) + if rs is not None: + return rs + extra_body = optional_params.get("extra_body") + if isinstance(extra_body, dict): + return extra_body.get("reasoningSummary") or extra_body.get("reasoning_summary") + return None + + +def strip_reasoning_summary_aliases_from_optional_params( + optional_params: dict, +) -> Tuple[dict, Optional[Any]]: + """Copy optional_params; remove reasoningSummary aliases from top-level and extra_body.""" + op = dict(optional_params) + rs_val = op.pop("reasoningSummary", None) + if rs_val is None: + rs_val = op.pop("reasoning_summary", None) + eb = op.get("extra_body") + if isinstance(eb, dict): + eb = dict(eb) + if rs_val is None: + rs_val = eb.pop("reasoningSummary", None) or eb.pop( + "reasoning_summary", None + ) + else: + eb.pop("reasoningSummary", None) + eb.pop("reasoning_summary", None) + if eb: + op["extra_body"] = eb + else: + op.pop("extra_body", None) + return op, rs_val + + +def strip_reasoning_summary_aliases_from_openai_completion_params( + non_default_params: dict, + optional_params: dict, +) -> None: + """Drop AI-SDK reasoning summary keys from chat completion param dicts (in-place). + + These aliases are not valid on OpenAI Chat Completions and may appear on + ``non_default_params`` or ``optional_params`` (including nested ``extra_body``). + """ + non_default_params.pop("reasoningSummary", None) + non_default_params.pop("reasoning_summary", None) + stripped, _ = strip_reasoning_summary_aliases_from_optional_params(optional_params) + optional_params.clear() + optional_params.update(stripped) + + def get_non_default_transcription_params(kwargs: dict) -> dict: from litellm.constants import OPENAI_TRANSCRIPTION_PARAMS diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 4358d0dc193..b4c2d0b9df2 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -757,6 +757,24 @@ def test_responses_api_bridge_check_gpt_5_4_tools_without_reasoning_stays_chat() assert model_info.get("mode") != "responses" +def test_responses_api_bridge_check_gpt_5_4_reasoning_summary_without_tools_routes_to_responses(): + """gpt-5.4+ with reasoning_effort + reasoningSummary but no tools should bridge (AI SDK).""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.4", + custom_llm_provider="openai", + tools=None, + reasoning_effort="medium", + reasoning_summary="auto", + ) + + assert model == "gpt-5.4" + assert model_info.get("mode") == "responses" + + @patch("litellm.completion_extras.responses_api_bridge.completion") def test_gpt_5_4_responses_bridge_preserves_reasoning_summary_dict( mock_responses_completion, @@ -794,6 +812,33 @@ def test_gpt_5_4_responses_bridge_preserves_reasoning_summary_dict( } +@patch("litellm.completion_extras.responses_api_bridge.completion") +def test_gpt_5_4_responses_bridge_merges_reasoning_summary_kwarg_without_tools( + mock_responses_completion, +): + """reasoningSummary without tools should route and merge into reasoning_effort dict.""" + mock_responses_completion.return_value = MagicMock() + + import litellm + + litellm.completion( + model="gpt-5.4", + messages=[{"role": "user", "content": "ok"}], + reasoning_effort="medium", + reasoningSummary="auto", + api_key="fake-key", + ) + + assert mock_responses_completion.called is True + optional_params = mock_responses_completion.call_args.kwargs["optional_params"] + assert optional_params["reasoning_effort"] == { + "effort": "medium", + "summary": "auto", + } + assert "reasoningSummary" not in optional_params + assert "reasoning_summary" not in optional_params + + def test_responses_api_bridge_check_handles_exception(): """Test that responses_api_bridge_check handles exceptions and still processes responses/ models.""" from litellm.main import responses_api_bridge_check From eed6985cd647bc396b931d2a58ac17abe0e75679 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 11 May 2026 05:25:06 +0000 Subject: [PATCH 58/85] Fix reasoning summary alias stripping --- .../llms/openai/chat/gpt_5_transformation.py | 12 +-- litellm/utils.py | 25 ++++--- .../llms/openai/test_gpt5_transformation.py | 74 ++++++++++++++++++- 3 files changed, 93 insertions(+), 18 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index eb7bb384a7e..a665ef65e5a 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -212,6 +212,12 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model: str, drop_params: bool, ) -> dict: + # AI SDK / Responses-style aliases; never valid on Chat Completions when not + # bridged to Responses API (see main.responses_api_bridge_check). + self._strip_reasoning_summary_aliases_for_chat_completions( + non_default_params, optional_params + ) + if self.is_model_gpt_5_search_model(model): if "max_tokens" in non_default_params: optional_params["max_completion_tokens"] = non_default_params.pop( @@ -224,12 +230,6 @@ class OpenAIGPT5Config(OpenAIGPTConfig): drop_params=drop_params, ) - # AI SDK / Responses-style aliases; never valid on Chat Completions when not - # bridged to Responses API (see main.responses_api_bridge_check). - self._strip_reasoning_summary_aliases_for_chat_completions( - non_default_params, optional_params - ) - # Get raw reasoning_effort and effective effort level for all guards. # Use effective_effort (extracted string) for xhigh validation, "none" checks, and # tool/sampling guards — dict inputs like {"effort": "none", "summary": "detailed"} diff --git a/litellm/utils.py b/litellm/utils.py index d473446b180..f61a47c4011 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9494,14 +9494,17 @@ def get_non_default_completion_params(kwargs: dict) -> dict: def peek_reasoning_summary_aliases(optional_params: dict) -> Optional[Any]: """Read AI-SDK-style reasoning summary from optional_params or nested extra_body.""" - rs = optional_params.get("reasoningSummary") or optional_params.get( - "reasoning_summary" - ) + rs = optional_params.get("reasoningSummary") + if rs is None: + rs = optional_params.get("reasoning_summary") if rs is not None: return rs extra_body = optional_params.get("extra_body") if isinstance(extra_body, dict): - return extra_body.get("reasoningSummary") or extra_body.get("reasoning_summary") + rs = extra_body.get("reasoningSummary") + if rs is None: + rs = extra_body.get("reasoning_summary") + return rs return None @@ -9511,18 +9514,18 @@ def strip_reasoning_summary_aliases_from_optional_params( """Copy optional_params; remove reasoningSummary aliases from top-level and extra_body.""" op = dict(optional_params) rs_val = op.pop("reasoningSummary", None) + snake_rs_val = op.pop("reasoning_summary", None) if rs_val is None: - rs_val = op.pop("reasoning_summary", None) + rs_val = snake_rs_val eb = op.get("extra_body") if isinstance(eb, dict): eb = dict(eb) + eb_rs_val = eb.pop("reasoningSummary", None) + eb_snake_rs_val = eb.pop("reasoning_summary", None) if rs_val is None: - rs_val = eb.pop("reasoningSummary", None) or eb.pop( - "reasoning_summary", None - ) - else: - eb.pop("reasoningSummary", None) - eb.pop("reasoning_summary", None) + rs_val = eb_rs_val + if rs_val is None: + rs_val = eb_snake_rs_val if eb: op["extra_body"] = eb else: diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index ebf7681f2f3..be079ee7086 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -4,7 +4,11 @@ import litellm from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.llms.openai.openai import OpenAIConfig -from litellm.utils import _is_explicitly_disabled_factory +from litellm.utils import ( + _is_explicitly_disabled_factory, + peek_reasoning_summary_aliases, + strip_reasoning_summary_aliases_from_optional_params, +) @pytest.fixture() @@ -1007,6 +1011,74 @@ def test_gpt5_search_drops_unsupported_params(config: OpenAIConfig): assert "tools" not in params +def test_gpt5_search_strips_reasoning_summary_aliases(gpt5_config: OpenAIGPT5Config): + """Search models still strip Responses-only reasoning summary aliases.""" + non_default_params = { + "reasoningSummary": "auto", + "reasoning_summary": "ignored", + } + optional_params = { + "extra_body": { + "reasoningSummary": "auto", + "reasoning_summary": "ignored", + "metadata": "ok", + } + } + + params = gpt5_config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5-search-api", + drop_params=False, + ) + + assert "reasoningSummary" not in non_default_params + assert "reasoning_summary" not in non_default_params + assert "reasoningSummary" not in params + assert "reasoning_summary" not in params + assert params["extra_body"] == {"metadata": "ok"} + + +def test_reasoning_summary_alias_helpers_preserve_falsy_and_strip_all_aliases(): + optional_params = {"reasoningSummary": False, "reasoning_summary": "ignored"} + + assert peek_reasoning_summary_aliases(optional_params) is False + stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params( + optional_params + ) + + assert rs_val is False + assert stripped == {} + + optional_params = { + "extra_body": {"reasoningSummary": False, "reasoning_summary": "ignored"} + } + + assert peek_reasoning_summary_aliases(optional_params) is False + stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params( + optional_params + ) + + assert rs_val is False + assert stripped == {} + + optional_params = { + "extra_body": { + "reasoningSummary": "auto", + "reasoning_summary": "ignored", + "metadata": "ok", + } + } + + assert peek_reasoning_summary_aliases(optional_params) == "auto" + stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params( + optional_params + ) + + assert rs_val == "auto" + assert stripped == {"extra_body": {"metadata": "ok"}} + + # GPT-5 unsupported params audit (validated via direct API calls) def test_gpt5_rejects_params_unsupported_by_openai(config: OpenAIConfig): """Params that OpenAI rejects for all GPT-5 reasoning models.""" From 0ac923c6b633e2ba274463ffa42fba84230bda35 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 11 May 2026 05:39:50 +0000 Subject: [PATCH 59/85] Fix GPT-5 reasoning summary alias stripping --- .../llms/openai/chat/gpt_5_transformation.py | 17 ------- litellm/main.py | 10 ++++ litellm/utils.py | 16 ------ .../llms/openai/test_gpt5_transformation.py | 50 ++++++++++--------- 4 files changed, 37 insertions(+), 56 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index a665ef65e5a..9ccb2e1c267 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -6,7 +6,6 @@ import litellm from litellm.utils import ( _is_explicitly_disabled_factory, _supports_factory, - strip_reasoning_summary_aliases_from_openai_completion_params, ) from .gpt_transformation import OpenAIGPTConfig @@ -195,16 +194,6 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if param not in non_supported_params ] - @staticmethod - def _strip_reasoning_summary_aliases_for_chat_completions( - non_default_params: dict, - optional_params: dict, - ) -> None: - """Remove Responses-style reasoning summary keys; invalid on Chat Completions.""" - strip_reasoning_summary_aliases_from_openai_completion_params( - non_default_params, optional_params - ) - def map_openai_params( self, non_default_params: dict, @@ -212,12 +201,6 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model: str, drop_params: bool, ) -> dict: - # AI SDK / Responses-style aliases; never valid on Chat Completions when not - # bridged to Responses API (see main.responses_api_bridge_check). - self._strip_reasoning_summary_aliases_for_chat_completions( - non_default_params, optional_params - ) - if self.is_model_gpt_5_search_model(model): if "max_tokens" in non_default_params: optional_params["max_completion_tokens"] = non_default_params.pop( diff --git a/litellm/main.py b/litellm/main.py index 6a3e5db2057..94915503803 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1696,6 +1696,16 @@ def completion( # type: ignore # noqa: PLR0915 encoding=_get_encoding(), stream=stream, ) + elif ( + custom_llm_provider == "openai" + and OpenAIGPT5Config.is_model_gpt_5_model(model) + ) or ( + custom_llm_provider == "azure" + and litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model(model) + ): + optional_params, _ = strip_reasoning_summary_aliases_from_optional_params( + optional_params + ) if custom_llm_provider == "azure": # azure configs diff --git a/litellm/utils.py b/litellm/utils.py index f61a47c4011..daac5234003 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9533,22 +9533,6 @@ def strip_reasoning_summary_aliases_from_optional_params( return op, rs_val -def strip_reasoning_summary_aliases_from_openai_completion_params( - non_default_params: dict, - optional_params: dict, -) -> None: - """Drop AI-SDK reasoning summary keys from chat completion param dicts (in-place). - - These aliases are not valid on OpenAI Chat Completions and may appear on - ``non_default_params`` or ``optional_params`` (including nested ``extra_body``). - """ - non_default_params.pop("reasoningSummary", None) - non_default_params.pop("reasoning_summary", None) - stripped, _ = strip_reasoning_summary_aliases_from_optional_params(optional_params) - optional_params.clear() - optional_params.update(stripped) - - def get_non_default_transcription_params(kwargs: dict) -> dict: from litellm.constants import OPENAI_TRANSCRIPTION_PARAMS diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index be079ee7086..840f2c75fde 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -1,6 +1,7 @@ import pytest import litellm +import litellm.main as litellm_main from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.llms.openai.openai import OpenAIConfig @@ -1011,32 +1012,35 @@ def test_gpt5_search_drops_unsupported_params(config: OpenAIConfig): assert "tools" not in params -def test_gpt5_search_strips_reasoning_summary_aliases(gpt5_config: OpenAIGPT5Config): - """Search models still strip Responses-only reasoning summary aliases.""" - non_default_params = { - "reasoningSummary": "auto", - "reasoning_summary": "ignored", - } - optional_params = { - "extra_body": { - "reasoningSummary": "auto", - "reasoning_summary": "ignored", - "metadata": "ok", - } - } +def test_gpt5_chat_strips_reasoning_summary_aliases_after_bridge_check( + monkeypatch: pytest.MonkeyPatch, +): + """Non-bridged GPT-5 chat calls strip Responses-only reasoning summary aliases.""" + captured_kwargs = {} - params = gpt5_config.map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model="gpt-5-search-api", - drop_params=False, + def fake_openai_completion(**kwargs): + captured_kwargs.update(kwargs) + return {} + + monkeypatch.setattr( + litellm_main.openai_chat_completions, + "completion", + fake_openai_completion, ) - assert "reasoningSummary" not in non_default_params - assert "reasoning_summary" not in non_default_params - assert "reasoningSummary" not in params - assert "reasoning_summary" not in params - assert params["extra_body"] == {"metadata": "ok"} + litellm.completion( + model="gpt-5", + messages=[{"role": "user", "content": "ok"}], + reasoning_effort="medium", + reasoningSummary="auto", + extra_body={"reasoning_summary": "ignored", "metadata": "ok"}, + api_key="fake-key", + ) + + optional_params = captured_kwargs["optional_params"] + assert "reasoningSummary" not in optional_params + assert "reasoning_summary" not in optional_params + assert optional_params["extra_body"] == {"metadata": "ok"} def test_reasoning_summary_alias_helpers_preserve_falsy_and_strip_all_aliases(): From 22e9fd12dfd46a7ee3eabf09a98556bc606614af Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 11 May 2026 11:12:24 +0530 Subject: [PATCH 60/85] Fix reasoningSummary for gpt-5 series as well --- litellm/main.py | 28 +++++++++---- tests/test_litellm/test_main.py | 72 +++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 8 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 94915503803..3fc1f2d4dd3 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -991,17 +991,29 @@ def responses_api_bridge_check( mode = "responses" model_info["mode"] = mode - # OpenAI/Azure gpt-5.4+ chat-completions calls that need Responses-only fields - # (e.g. reasoning summary) must be bridged. SDKs send ``reasoningSummary`` / - # ``reasoning_summary`` alongside ``reasoning_effort``; Chat Completions rejects - # those keys, so route when tools+reasoning_effort (original case) or when a - # reasoning summary is requested without tools. + # OpenAI/Azure GPT-5 chat-completions that need Responses-only fields (e.g. + # ``reasoningSummary`` in ``extra_body``) must be bridged; Chat Completions rejects + # those keys. + # + # - gpt-5.4+: tools + reasoning_effort (original) or any reasoning-summary alias. + # - Older GPT-5 names (e.g. ``gpt-5``, ``gpt-5.1``): bridge only when a reasoning + # summary alias is present with ``reasoning_effort`` (tools alone stay on chat). if ( custom_llm_provider in ("openai", "azure") - and OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) - and reasoning_effort is not None - and (tools or reasoning_summary is not None) and model_info.get("mode") != "responses" + and OpenAIGPT5Config.is_model_gpt_5_model(model) + and not OpenAIGPT5Config.is_model_gpt_5_search_model(model) + and reasoning_effort is not None + and ( + ( + OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) + and (tools or reasoning_summary is not None) + ) + or ( + not OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) + and reasoning_summary is not None + ) + ) ): model_info["mode"] = "responses" model = model.replace("responses/", "") diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index b4c2d0b9df2..c37f9fc26b4 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -775,6 +775,42 @@ def test_responses_api_bridge_check_gpt_5_4_reasoning_summary_without_tools_rout assert model_info.get("mode") == "responses" +def test_responses_api_bridge_check_gpt_5_reasoning_summary_routes_to_responses(): + """Bare ``gpt-5`` with reasoning_effort + reasoningSummary should bridge (not 5.4+).""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5", + custom_llm_provider="openai", + tools=None, + reasoning_effort="medium", + reasoning_summary="auto", + ) + + assert model == "gpt-5" + assert model_info.get("mode") == "responses" + + +def test_responses_api_bridge_check_gpt_5_tools_without_summary_stays_chat(): + """gpt-5 with tools + reasoning_effort but no summary should stay on chat.""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort="medium", + reasoning_summary=None, + ) + + assert model == "gpt-5" + assert model_info.get("mode") != "responses" + + @patch("litellm.completion_extras.responses_api_bridge.completion") def test_gpt_5_4_responses_bridge_preserves_reasoning_summary_dict( mock_responses_completion, @@ -839,6 +875,42 @@ def test_gpt_5_4_responses_bridge_merges_reasoning_summary_kwarg_without_tools( assert "reasoning_summary" not in optional_params +@patch("litellm.completion_extras.responses_api_bridge.completion") +def test_gpt_5_responses_bridge_tools_and_reasoning_summary( + mock_responses_completion, +): + """Bare gpt-5 with tools + reasoningSummary should bridge (OpenCode-style).""" + mock_responses_completion.return_value = MagicMock() + + import litellm + + litellm.completion( + model="gpt-5", + messages=[{"role": "user", "content": "ok"}], + tools=[ + { + "type": "function", + "function": { + "name": "apply_patch", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + tool_choice="auto", + reasoning_effort="medium", + reasoningSummary="auto", + stream=True, + api_key="fake-key", + ) + + assert mock_responses_completion.called is True + optional_params = mock_responses_completion.call_args.kwargs["optional_params"] + assert optional_params.get("reasoning_effort") == { + "effort": "medium", + "summary": "auto", + } + + def test_responses_api_bridge_check_handles_exception(): """Test that responses_api_bridge_check handles exceptions and still processes responses/ models.""" from litellm.main import responses_api_bridge_check From e74329db303ab13ff19e8335ccbbbe000bf6f0b1 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 11 May 2026 11:13:40 +0530 Subject: [PATCH 61/85] Fix greptile issue --- litellm/utils.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index daac5234003..80452e533b6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9493,18 +9493,20 @@ def get_non_default_completion_params(kwargs: dict) -> dict: def peek_reasoning_summary_aliases(optional_params: dict) -> Optional[Any]: - """Read AI-SDK-style reasoning summary from optional_params or nested extra_body.""" - rs = optional_params.get("reasoningSummary") - if rs is None: - rs = optional_params.get("reasoning_summary") - if rs is not None: - return rs + """Read AI-SDK-style reasoning summary from optional_params or nested extra_body. + + Uses key membership (not ``or`` chains) so falsy values like ``""`` are not skipped. + """ + if "reasoningSummary" in optional_params: + return optional_params["reasoningSummary"] + if "reasoning_summary" in optional_params: + return optional_params["reasoning_summary"] extra_body = optional_params.get("extra_body") if isinstance(extra_body, dict): - rs = extra_body.get("reasoningSummary") - if rs is None: - rs = extra_body.get("reasoning_summary") - return rs + if "reasoningSummary" in extra_body: + return extra_body["reasoningSummary"] + if "reasoning_summary" in extra_body: + return extra_body["reasoning_summary"] return None From 57ed2dad4bdacc982c069b8a83aa1c19ce6c47d5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 11 May 2026 05:51:51 +0000 Subject: [PATCH 62/85] Simplify GPT-5 responses bridge condition --- litellm/main.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 3fc1f2d4dd3..c324f982b84 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1005,14 +1005,8 @@ def responses_api_bridge_check( and not OpenAIGPT5Config.is_model_gpt_5_search_model(model) and reasoning_effort is not None and ( - ( - OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) - and (tools or reasoning_summary is not None) - ) - or ( - not OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) - and reasoning_summary is not None - ) + reasoning_summary is not None + or (OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) and tools) ) ): model_info["mode"] = "responses" From 1628886f4a3584903c2d39beb4d079a6b51bf1e5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 11 May 2026 06:01:35 +0000 Subject: [PATCH 63/85] Fix GPT-5 reasoning summary strip test path --- tests/test_litellm/llms/openai/test_gpt5_transformation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 840f2c75fde..d279b119efe 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -1031,7 +1031,6 @@ def test_gpt5_chat_strips_reasoning_summary_aliases_after_bridge_check( litellm.completion( model="gpt-5", messages=[{"role": "user", "content": "ok"}], - reasoning_effort="medium", reasoningSummary="auto", extra_body={"reasoning_summary": "ignored", "metadata": "ok"}, api_key="fake-key", From 055bdc35077c05314680c7c4444b8e6666852940 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 11 May 2026 11:35:56 +0530 Subject: [PATCH 64/85] fix(auth): harden JWT routing wildcard iss and merge list team_id claims Reject fnmatch wildcards on non-scope claims when the claim string contains whitespace so malformed iss values cannot match patterns like trusted.*. Merge every entry when team_id_jwt_field resolves to a list instead of keeping only the first element. Co-authored-by: Cursor --- litellm/proxy/auth/handle_jwt.py | 15 ++- litellm/proxy/auth/user_api_key_auth.py | 5 + .../proxy/auth/test_handle_jwt.py | 5 + .../proxy/auth/test_user_api_key_auth.py | 100 ++++++++++-------- 4 files changed, 77 insertions(+), 48 deletions(-) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index a270031c33c..5b116c142b2 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -227,8 +227,8 @@ class JWTHandler: def get_all_jwt_team_ids(self, token: dict) -> List[str]: """ Return team IDs from both the plural ``team_ids_jwt_field`` and the - singular ``team_id_jwt_field`` claim, as a deduplicated list preserving - plural-first order. + singular ``team_id_jwt_field`` claim (string or list of strings), as a + deduplicated list preserving plural-first order. Membership-reconciliation paths (SSO callback, JWT-bearer sync) need to consider both claim shapes. Reading only the plural field — as @@ -249,9 +249,14 @@ class JWTHandler: default=None, ) if isinstance(singular, list): - singular = singular[0] if singular else None - if singular and singular not in team_ids: - team_ids.append(singular) + for item in singular: + if item is None: + continue + sid = str(item) + if sid and sid not in team_ids: + team_ids.append(sid) + elif singular and str(singular) not in team_ids: + team_ids.append(str(singular)) return team_ids def get_end_user_id( diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 2f61a40bc9a..213c68f1db2 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -219,6 +219,11 @@ def _routing_selector_matches_claim( def _selector_matches_claim(selector: str, claim: str) -> bool: # NOTE: wildcard matching is case-sensitive (fnmatch.fnmatchcase). if "*" in selector or "?" in selector: + # Without scope splitting, do not let `*` span whitespace: a malformed + # iss like "trusted.example.com evil.com" must not match "trusted.*". + # Scope uses split_space_delimited so each claim token is checked separately. + if not split_space_delimited and any(ch.isspace() for ch in claim): + return False return fnmatch.fnmatchcase(claim, selector) return selector == claim diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index c09c303deee..90c5d4f4fc5 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -524,6 +524,11 @@ def test_get_all_jwt_team_ids_unions_singular_and_plural(): "b", ] + # singular field as multi-element list (some IdPs) — merge all, preserve plural-first order + assert jwt_handler.get_all_jwt_team_ids( + {"team_id": ["primary", "secondary"], "teams": ["a"]} + ) == ["a", "primary", "secondary"] + # neither populated assert jwt_handler.get_all_jwt_team_ids({}) == [] diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 4b0f6c0ea2f..a6483403c27 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -587,6 +587,14 @@ def _assert_get_api_key_with_custom_litellm_key_header( False, False, ), + # Wildcard iss must not match space-containing claim strings (fnmatch * spans spaces) + ( + "trusted.*", + "trusted.example.com attacker.example.com", + False, + False, + ), + ("trusted.*", "trusted.example.com", True, False), ( ["issuer-a.example.com", "issuer-b.example.com"], "issuer-b.example.com", @@ -667,7 +675,10 @@ def test_routing_selector_matches_claim_parametrized( aud=["api://litellm", "api://fallback"], path="oauth2", ), - {"iss": "oauth-issuer.example.com", "aud": ["api://other", "api://litellm"]}, + { + "iss": "oauth-issuer.example.com", + "aud": ["api://other", "api://litellm"], + }, True, ), # All provided selectors are AND-ed. @@ -1738,20 +1749,21 @@ class TestJWTOAuth2Coexistence: mock_request.headers = {"authorization": f"Bearer {jwt_token}"} mock_request.query_params = {} - with patch( - "litellm.proxy.proxy_server.general_settings", general_settings - ), patch("litellm.proxy.proxy_server.premium_user", True), patch( - "litellm.proxy.proxy_server.master_key", "sk-master" - ), patch( - "litellm.proxy.proxy_server.prisma_client", None - ), patch( - "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", - new_callable=AsyncMock, - return_value=mock_oauth2_response, - ) as mock_oauth2, patch( - "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", - new_callable=AsyncMock, - ) as mock_jwt_auth: + with ( + patch("litellm.proxy.proxy_server.general_settings", general_settings), + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.master_key", "sk-master"), + patch("litellm.proxy.proxy_server.prisma_client", None), + patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + return_value=mock_oauth2_response, + ) as mock_oauth2, + patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + ) as mock_jwt_auth, + ): litellm.proxy.proxy_server.jwt_handler.update_environment( prisma_client=None, user_api_key_cache=DualCache(), @@ -1809,20 +1821,21 @@ class TestJWTOAuth2Coexistence: mock_request.headers = {"authorization": f"Bearer {jwt_token}"} mock_request.query_params = {} - with patch( - "litellm.proxy.proxy_server.general_settings", general_settings - ), patch("litellm.proxy.proxy_server.premium_user", True), patch( - "litellm.proxy.proxy_server.master_key", "sk-master" - ), patch( - "litellm.proxy.proxy_server.prisma_client", None - ), patch( - "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", - new_callable=AsyncMock, - ) as mock_oauth2, patch( - "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", - new_callable=AsyncMock, - return_value=mock_jwt_result, - ) as mock_jwt_auth: + with ( + patch("litellm.proxy.proxy_server.general_settings", general_settings), + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.master_key", "sk-master"), + patch("litellm.proxy.proxy_server.prisma_client", None), + patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + ) as mock_oauth2, + patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + return_value=mock_jwt_result, + ) as mock_jwt_auth, + ): litellm.proxy.proxy_server.jwt_handler.update_environment( prisma_client=None, user_api_key_cache=DualCache(), @@ -1873,20 +1886,21 @@ class TestJWTOAuth2Coexistence: mock_request.headers = {"authorization": f"Bearer {jwt_token}"} mock_request.query_params = {} - with patch( - "litellm.proxy.proxy_server.general_settings", general_settings - ), patch("litellm.proxy.proxy_server.premium_user", True), patch( - "litellm.proxy.proxy_server.master_key", "sk-master" - ), patch( - "litellm.proxy.proxy_server.prisma_client", None - ), patch( - "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", - new_callable=AsyncMock, - return_value=mock_oauth2_response, - ) as mock_oauth2, patch( - "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", - new_callable=AsyncMock, - ) as mock_jwt_auth: + with ( + patch("litellm.proxy.proxy_server.general_settings", general_settings), + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.master_key", "sk-master"), + patch("litellm.proxy.proxy_server.prisma_client", None), + patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + return_value=mock_oauth2_response, + ) as mock_oauth2, + patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + ) as mock_jwt_auth, + ): litellm.proxy.proxy_server.jwt_handler.update_environment( prisma_client=None, user_api_key_cache=DualCache(), From 7524c4022e49f79aa0d4440d10e0212da9e74de8 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 11 May 2026 18:56:48 +0530 Subject: [PATCH 65/85] dummy change --- litellm/llms/openai/chat/gpt_5_transformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 9ccb2e1c267..12a2d64ee6b 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -7,7 +7,7 @@ from litellm.utils import ( _is_explicitly_disabled_factory, _supports_factory, ) - + from .gpt_transformation import OpenAIGPTConfig From aa1f57fff85c01b541b2fac17210fa579361fb5c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 11 May 2026 20:41:10 +0530 Subject: [PATCH 66/85] fix black and github mock test --- .../llms/openai/chat/gpt_5_transformation.py | 2 +- .../test_github_copilot_transformation.py | 33 ++++++++++--------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 12a2d64ee6b..9ccb2e1c267 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -7,7 +7,7 @@ from litellm.utils import ( _is_explicitly_disabled_factory, _supports_factory, ) - + from .gpt_transformation import OpenAIGPTConfig diff --git a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py index 678aa6b56c1..45ce5d58405 100644 --- a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py +++ b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py @@ -94,27 +94,33 @@ def test_github_copilot_config_get_openai_compatible_provider_info(): @patch("litellm.llms.github_copilot.authenticator.Authenticator.get_api_key") +@patch("litellm.main.openai_chat_completions.completion") @patch("litellm.llms.openai.openai.OpenAIChatCompletion.completion") -def test_completion_github_copilot_mock_response(mock_completion, mock_get_api_key): +def test_completion_github_copilot_mock_response( + mock_class_completion, mock_instance_completion, mock_get_api_key, monkeypatch +): """Test the completion function with GitHub Copilot provider.""" - # Mock the API key return value + # Force chat path through the patched openai_chat_completions instance even if + # a previous test left EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER set in the env. + monkeypatch.delenv("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER", raising=False) + mock_api_key = "gh.test-key-123456789" mock_get_api_key.return_value = mock_api_key - # Mock completion response mock_response = MagicMock() mock_response.choices = [MagicMock()] mock_response.choices[0].message.content = "Hello, I'm GitHub Copilot!" - mock_completion.return_value = mock_response + # Patch both the class method and the live module-level instance to survive + # conftest module reloads that can swap which class object is in use. + mock_class_completion.return_value = mock_response + mock_instance_completion.return_value = mock_response - # Test non-streaming completion messages = [ {"role": "system", "content": "You're GitHub Copilot, an AI assistant."}, {"role": "user", "content": "Hello, who are you?"}, ] - # Create a properly formatted headers dictionary headers = { "editor-version": "Neovim/0.9.0", "Copilot-Integration-Id": "vscode-chat", @@ -128,19 +134,16 @@ def test_completion_github_copilot_mock_response(mock_completion, mock_get_api_k assert response is not None - # Verify the get_api_key call was made (can be called multiple times) assert mock_get_api_key.call_count >= 1 - # Verify the completion call was made with the expected params - mock_completion.assert_called_once() - args, kwargs = mock_completion.call_args + # Exactly one of the two patched targets should have been used. + invoked = [m for m in (mock_class_completion, mock_instance_completion) if m.called] + assert len(invoked) == 1 + invoked[0].assert_called_once() + _, kwargs = invoked[0].call_args - # Check that the proper authorization header is set assert "headers" in kwargs - # Check that the model name is correctly formatted - assert ( - kwargs.get("model") == "gpt-4" - ) # Model name should be without provider prefix + assert kwargs.get("model") == "gpt-4" assert kwargs.get("messages") == messages From b1508161ecb5bd2695fb677101cca8755d489bcd Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 11 May 2026 15:25:40 +0000 Subject: [PATCH 67/85] Preserve reasoning summary without effort --- litellm/main.py | 2 ++ tests/test_litellm/test_main.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/litellm/main.py b/litellm/main.py index c324f982b84..29a9ffc84f8 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1684,6 +1684,8 @@ def completion( # type: ignore # noqa: PLR0915 "effort": eff, "summary": rs_val, } + else: + optional_params["reasoning_effort"] = {"summary": rs_val} return responses_api_bridge.completion( model=model, diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index c37f9fc26b4..76336a91fc3 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -875,6 +875,30 @@ def test_gpt_5_4_responses_bridge_merges_reasoning_summary_kwarg_without_tools( assert "reasoning_summary" not in optional_params +@patch("litellm.completion_extras.responses_api_bridge.completion") +def test_responses_bridge_preserves_reasoning_summary_without_effort( + mock_responses_completion, +): + """Reasoning summary should survive responses routing even without effort.""" + mock_responses_completion.return_value = MagicMock() + + import litellm + + with patch.object(litellm, "route_all_chat_openai_to_responses", True): + litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "ok"}], + reasoningSummary="auto", + api_key="fake-key", + ) + + assert mock_responses_completion.called is True + optional_params = mock_responses_completion.call_args.kwargs["optional_params"] + assert optional_params["reasoning_effort"] == {"summary": "auto"} + assert "reasoningSummary" not in optional_params + assert "reasoning_summary" not in optional_params + + @patch("litellm.completion_extras.responses_api_bridge.completion") def test_gpt_5_responses_bridge_tools_and_reasoning_summary( mock_responses_completion, From 480142533658915ea7eb56bb1af0ee0fcb3d385a Mon Sep 17 00:00:00 2001 From: superpoussin22 Date: Mon, 11 May 2026 17:49:53 +0200 Subject: [PATCH 68/85] Add gpt-realtime-2 model pricing --- model_prices_and_context_window.json | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e66ad8e0cf6..70b065aa918 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -21109,6 +21109,38 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "gpt-realtime-2": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "openai", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1.6e-05, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, From 8631d97e3e95c793bd6cacd4cff62c4392bd93eb Mon Sep 17 00:00:00 2001 From: superpoussin22 Date: Mon, 11 May 2026 17:51:20 +0200 Subject: [PATCH 69/85] Add gpt-realtime-2 model pricing and capabilities --- ...odel_prices_and_context_window_backup.json | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 76d2d35a53a..8df25d2c9b5 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -21104,6 +21104,38 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "gpt-realtime-2": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "openai", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1.6e-05, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, From 9bc90f6d1b73b178b9cc6fba48cbcea61313d784 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 11 May 2026 21:27:59 +0530 Subject: [PATCH 70/85] chore: remove accidental .evidence screenshot from repo (#27633) Co-authored-by: Cursor --- .evidence/main_header_repro.png | Bin 61972 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .evidence/main_header_repro.png diff --git a/.evidence/main_header_repro.png b/.evidence/main_header_repro.png deleted file mode 100644 index d71ab04d02a56b3ad8f2e8015bf53ca13f8cd8d3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 61972 zcmeFYbyQUE-!6;^h_ryTv`9&p4ALbaInvT89nuOT4TE$e-3;9!Al(fTLw5}gL%iGg zKIi$JC*J3*b^biFX1T`Ap4oSN>b|b)6Zlp^3iA=kBNP-AOc`l$WfYVL=_n|7U!vUw zt{_i?ZBS4UGcw{Lsvnbf=3KP#*C|mC7_7dsxTD{_>vPXIFTf|DrV^`SLc?-9e?n+0 zcf77reLLUDa&F@Db82c1)$b3m?tkf_NVh@@zq;W|)a$9K4~O6i@5i#~#r8T3dZyY9 z(qYF6hSgQwxdS}$*URi;B|+fNN8sP??_8*V|I>x`#^LYBPa*&B^6#biKU|Z5j<~g_ z<=oN|VmY|Zw0I5-`_r*yh3=27w_g8U+1DH-n-*ADIp(aS>{jp(r&)zq*B!jw^Q#2W zAB3c^bUfXDmd1{THIq~#5{lNPTNk-xUK*6`3RwVm@~k;?Pb4~>CV4V$k2VL#|GKSi zD=N?sdutBMv$MY`YL&x8B(&%uGl&mqkeyb$J6f6-Ltm+HJ!`cT+4!7bK;)@XUIhtXm>%P0Bhc&t?F1sUvh&# zObA}3r6<}I96rh~qgWRfH?BGJI=rWApgkYC08U#k!!t-=RsZwMaFLOOwFTRihit-o zTprmmN`kG66%#xqCuICf>%q!Db&gcqS={ZAuwY>!(`-8dy>TA9f5x(H4)|Zr-Kk&i3r~()s)lVt}p15J*vI zO&Tr!ax4^-PazQ#T>iybzNNnI9l@wlIWE1fM>}J)^So6GgI?slAk=LNUsJuFGTXg% zj!S>@PCZYDS1s#ExawGCo(n%&Z6GqE(;BW?x#nds)b0qdYSj@5r+wS_Wwz#1 z{Go-Te2ntj+D1!VSA>*~9O~9OzXr@u?@LPgzF<|T1gdC&(_ork)xfZ6b*xt(ZaMRx z|m=_Hy%?Dp;+o5B_5~BLhymh2c6=Jg>`Hb>AxP zO(PvFm&;)H!B#Inm3Z6tS=X^wAMm%o_m93?d-g^~t*x*;M*^G*CAP^GFgN;1KM{7# z@YD5zsleQ;&!e}XSVJNNeQFs8OKO9VR;hEc+JSO|-Swo>N$(`CWJ&BYF$a8WX1n%x zMkik8F=35433%Zv9_JgG0$Di&o?Wexw<0R zCQC7P8#*AAH;&a9g0NvoI8h4&Iu(QI87<^#)jp4f%ImI_2f(BYIlX5<89rXg(Rk#U z=0^MmWXmQW=($kL5e(Tbt_y!|toXa611rL;}sdq>vxHcO`fe*f9xlw8Rwi{ikH<2Qy zA1VBIHH6IN9ij-Dnj76yCi_0D=`^ZxJMCVED0qD=f5rsta0)89;^W&yoWq;Bf+5df zDH|-V{T&(LJ04vMfmlFD;W(TcY}k4s8t`FXX>%|Gl-aDG>MgoaJE>2EPE~8OtUC(= z;>l+}5v{F5obrK2bSkRm!9h<_DKwd24Ct$-CV0b~NwPbJqQlV2TYP^S2H0~3wju$T zNNuG_wUWbb3SB>PwWy=_;YNrs{H(>cni&`57wGDmmmeKkTGZ1Lf5ah(D1Bdh?ny``u9U@n zqAWjIQ$Y}<*MN?d0UVvH5lwY*IYg{de_@@(%+yI*#&P275VNV-^?Q}v=T zkOwGDa%RP-82G}9kU^_{fGz9z4;DUWM4Niaf3X<)kQRqHRZmGdF8O?8IBU+_4!3n5 z@EUrEt}SWxM*D6vZ3)eviG8Y*Mtm-VgqQg=5Vhn4!A(>2cvePw*aXf!Ho1^KSP zHpR9ghgy?V1#P@V1Cl2<4ws-ZN|WL}h56~KC0Co(w4xW2yKuNcK$Mv4u>X&P-ZMpO zv#jBck;-Oi5uQH()p$}kA%NBih z9 z%;75>9G>ctVdcOZ;L%!^^pKQ-Qdd_c5LIUK%xg6kPo?1S%VSomAC_x8I&xAH1Zw!5 z6mtxPIEAKdHjAHG#hORitJU5rD-92o?(vUZ_tHVa&u$x@aK}9?D9-or+#hMD^iNIN zMNpqdMbCWy+N^zJp-4NUjMd(j$^5p9;d*A%lR4;-AhlKmeNn-MB-g zFET-mGu|zrmfj@Z!TH%{)}(JMV$QOCMb6guKqd{{GH#lNN3BU?K7B^z11T!l!Wk1} zNn>dZFH`%A@%K{pUzsH|spXW;_Q&tl^=-98t>6vNm~5kuFYX*ir)7TkZQ7$rRf zn1Z`^amcHxfeyns!`ndn!yrghVAS7Hev6Ns7$>u#oNXv|as!^dbj2K&2_jI%wv8V4)~-onLyNl&$M(kSO-HMovyeL;Wis;c&- z3qDoAR2!=fb)#@>I5xzB)s?+u;9k2@*g7L;o}1!eIbAAlh+3a5TLR2$IuI&$#!_45 zWe{5L&Xv8@69je{NYN9hX>$L@LH~D%W>tJ!KOc_FcTSO4v%wH-OkMcqOhEF!CM0jD z@)DKpW4Vm-jUg{}JP~9s@9gZWHh`f2V>^jYD4#QZUq@s*Uf-2cnD6oYmL+%F2Of@F z{(HxHA9n{ADiuVLR=8h(e;4jDpkQ&)z3O=!__tbqy$0OS8UNrqGuBmO5y=pH)>upV ztWoFh_g@DOJw5v;J)xN0A8{M`RqFqGy{2jVKdP+FLd?T(X{5-7F2HZ4&1}u$VqM^p zQ#4$ZM{8`4u`n}kSl(4+HW`JH3qJ7o!zAOXLEL*!OjSbF=p0qUeq-n>*4Ee8uaRv7 z@mgBiA|xYOo34j)alj&Tqo0Ye6+F&}Xj2E=3rw-;)qclvM0IMF&2oT>;C1GkSV@d;5Mw4=xf5YlLDyvk`;% zx#$u0YmZbh&O2R?*`f4g`(<}_r8Pw%cvLt3<0dzDXATGP$7(3=Wkh}o+S)lVK$zor zdH1GOD4-0xb0Q8$+owg%)^J%gbYN~Wgp_A}IZ)Hc``L2c1|-Tu5!C4+mknr3@cR&t2e^fxS}Qsw42%-$sPG)fJokR&9y*49@0+K>x7SLByP;Ed#}6%>5tg_85{%s19IHMP`L zT8@p46ESNiMJASCb`ogB$A{ojfV^7!3d7)ZVPrg+-@bj)RDY{_Oer|fIXT%ilv`87 zM}0Nlat|3C6m&mue0-eYO~Oc9o0yao1B0L-2glNJH~yC|Uv^wXQpJLw-<4}g)Bk+u z&Py2UocJEyQr(GVab2Ke?cX|2#kdWi= zpFYVE4Pe~11T~T8-O=COQj?XHjm&=YA)^4fg2-SY<@Go}_zZgW)hCmf9gT`UJDbyV zaC5djFE@ANAl2K<+(JH;-J$0POc~yYANF#``gye1!nc+qB9;_hQ)iD^&c*j;0(iZ} z?KU&=m_!dA{63C5@;A)uEsVameOrDth97Zy0sn}yVVLwpv0JjI>+I=)TbBVD?jJ`dBZus&;oC5eSLF& z`{Spst{13h8K-;mbV>KFKZ)txZ*{CGy(*FQWk>BmV;m7I$0?Qa;<+B`n)-MJ2^?T$ zO-Rrr-rRH%+fM>HMf9#d!V4G;3!xEC%n0-%I%dE^)C-_+f%5eFTRJ93o5`1jV>Bsv zZwkI8@&fCmqN+MuXJ{(O#AMN!_I3Ci9lw93>e_rijyQ@C`mhk5ay7 z)){zK{)iJcp8_SB@>D zOe_En*Y0FzpWmpfBG)@KbYQaJ#N%!%bLDq>a)dwoWvm{isD%-%0at+K=i@1imIzoS zDWWOUdEx$`ydXxVeaTvd)q89rb zl+@Loot|aE)s>X=^{1^7WyZd!*!<4t;R`m5$Gg%9#BQ_x_23x|jr<#V z?;B%Z_cMz8{QRt}EIf2z|DW=^g@lDE7>S>!=rl)1MV((_o?EXmD_I7n{Cf+yi=-ao zwJsNdt4)zvvKAKuQS^AhE2`;pON24gjv(Lh&L*xGTS$qZ-4KyTUq+q;**>HyC9xZ! zU`8r%;P(ATw~EL0)#=R*<0r4CYjmp5lkdj;&ZGOXW1eH~nJ;;W20)?P%ex=mzmH2! zrfZlV7)Zgvqa>1Qd~$RphufT%tfTF%QGM#A8r76?lb+RUwY$@Bc{y9>Di;39Gv2rw;EON``Q5`>5Bt2D2$|O5UddzASBfkcZ#NSyOi&Np`M>)mfBn2()zuTB)Yc3WFp3kwSkrLb~xvT#M~oE)ei zR&x#PjO?X_g*%80Db%xl28g4BGp-f#T*=B@raNQmsc2UlsuCR?9l{Qw`ST|pdS7q7 z4Un4h33_##6&CDk`KPC6s;wDW810eptQ11 zY15y;{gmmmC+-$$b$jR9yL#u{t>o|-z(g3ky9HgWXnsvHd)~ldDcl)%gWG>Od1%6c zlYbulgV=hrH!z^#$(+=NhF6sxos(zNqYQa@g*P0`UUcQKtcPb@c0L{ylu~G=<<$(3 z$CL;OpsevVl(N8tcDkP&K0ror;5Db+gP}t1qsGQWWoFWv1qn&L~v$&|gK z4&XoJJQdT?5kl<@W7BJzD!EyYidqReJggWS8_iJw)p$2c(i()-aozLzU(TLzBgM43 zy80D&v2t>Ia~rIt-XtT7SLk}N1Yi_;x3|3bX=`tLdwajc^hUkY!xR}D>wBXivD$M& z%002YwW*BZL1D9g`r(L>dA8Ej#zXt6-i48Uc(|6I|37Yx;T@q@LjKUF=Efh2va=<1 zN@dW@$Md75-DRXdGQJz=?WKX_21MNR3_qsvYfAA_H!=A-q^E(e+ZeIKpd;hderXXX ze{$e#&0KGk9M5U24u;tTK-?&m4;4r z2MlvNK6G?)fCh!w?o4%0O|5J!iZMIHNYj-U{$2aXC{fz#5dfx+7t++4UBSfuOtKB|Fw_t~YqI=sxR0K6^G z>)_Mi`luB3TCSY%vjE@KK(VW0ER@i^P1ssj_W&WIP%0W`h6jiWa=Ynm@j&mVB6BSG z{C4*3LD93@uQUTmHw6t1RznMEor=kO`@F9+XMYT)-J#WjtjW}b;0+zRlid4x_w;DD z*<`nlRY75EYs*LR2!hiBSJHUFhPn6lxfKT(f{LaBLdTxzWN zxdjXQVXP|W@Q&!~F*~jMm=?wzVXF0{(xqsaBU5em*C(^uVP|z{>p1xM$?tD@mC zVzxn|7f0t+A$yK5s$vGAMsudh!bE`p-FCK) z6=hWlvTrgcrkplj7sW2jyEa&e2Kb~K>{fG(i~?9u)~(s)#b!iA#6d|3@XqkZE}AJp z)N7R6%%d6AV#M%s-BOn*X)34x)U4E>3DT3~xf6g-l@T-Y#);&mnXS3`X13l$r;xj* zs&?Nx;vqJEs0e zRHsz`>5-`weTVFOJ;-IpZil|vx<3P>jQ;-pyY1-2E=A)j zC#RFBs3Au-cMC<5 zILfeJC!qxva;bRc7vvuv9m)0fjA>S%9vmm}nFsxFp`yRUrQxBMl#wxSS)zOR&=0U` z8YyTVB2jCLs8!4#T2fM?`eE*;&G+xRjjy(6l{~K@o?xq|JQt(9_=a?V&qr{W9lU4T zZvt{QAz_Szcbp=c>(%oZH#ubntlwuYJdx3%sPEf%$oOlGnbh<1^5z~rGDs~Kf1yEL zNqfTpx%|=jtDX-bS-w5Exh^X!JFlsM(aTdqUO|InRUr{eytdz}q0sz%3!uuXJtD#p zHOQbll7VG~n2RTv68`r8xY$_aE6~7vQ(GAU6ALpl5H{rBypcpA>+5QRb(n5j_?~gq z{L9Yx`1(kCu;+Wa&s?M4J}G>RjZt7%?ZIwrYMhmmN_BWV$;eB1_xvd_`fHkz^V~o{bi?;JXw;6;)*1BU|i8PWOwI5tv zU46}v9E=l^3OBh)ZwKsv9->`u|D_H){@ged`*r61B{Rl;)tm!&7eS}J2FpR+ExOdf zX19y|v^FU;yb>#^ymU%-R-J~IAG_W{byENv$fFjQgkVtUOQUrU4no~e3k`D=P@!aW z0AvH!Yj$?(Dp{;_QPmUHhZB(cv3qxz7!=+&C;B|(OmI4Q7!n$)QEE}Su(~~oe%1=Q z))e`dJ-RyE-H0pNw^ePR4aPCMfdwiBR7JC94xgvfS~JZ^PZdOUm)TB*y&Bl z>`a1!OlHdE*3st19h`e^RLjgPF>=1atJ3HOl4zbwfFl~a> z)%~Nxst@zWuZq{AXA-?M&VUj0^zDS`nXTp zBOVRl8e}#IyRy2}Kb)!N%vP*-r8)4_)ipL=t@pmUFuZNb?LuSU%D@V_C+dr|SoT;( z`6St)G+FF=>k3d+CSBz&scOPS*LKq*NTsjx5;*2X$d;t*;oFT)JeSF`nG0IPi$J!=`cfdpBP?sKJ@pCk5A-nZ4#&3 zrOLANW2U3aiZmTiNIsR{t)3{pHJ*V9X4Sd;8tA!|j7vgWldo@u@EhJ%*4ICyXQ@d_ zCGIoQ=M?Lh_Fv5Z_#v)P{F%o0oc~i?f}#{TcZZ&i{Pk@7XeC_0ZR2gufDSE3no~)`V3=fIpyLQVYT)AwKm64m;qr+dw()c+57p~8dF8rS1DZfxooX^)i&2oNOe(Z z0F=CU%4C>1M?jheunJ62VnNo8wb1nR^q>1pu93$ph~>aY1}O+U(&Fi_;T{mT>S^Cu zz@Ou(1EPdt>MQFhGJkP$MLAvsUj*6)-sUrMWCN zHa0nVa?;LDS69F&gW?lH!ZNsor)H&v6ZgSdDUJVJz02ez@4IT}(1f~6pD)1<_73W| zHPlR;k+N%Jl`|!aG*-L2+}zxp93#LPja7$_?dtl&zan~&GXT7;(m#r+EC4eyGRh}# zRE8Gs0pLNE)$GAy8|vl3^OQzp;idU5v^PdmLkmhGirQCKr&Y9~gW>+(=4DmJ-LN1U)H!M8Q7tt0j;8|;UNhuC{kyK{{Nl2|9> zRH>bY%y-j-W^<{>u&l0srtU#zO;qml@V}Il2GCxN$7oYk#wxqjPo*Ba{kN#2VZ3l+ zv6+$8E3Z~@Qz9aH`J#~Hs2RBJf#6jYZ$+yqvL??6>>J76M1%{t4H$7jy^p!E2Avx z>h3N=8fTOTW1J|cV> z1wn(imsV8W8?$3#($W_Dzi(S{=!^BK(?iZLFVL;k7Um{%ioUcL+T;MKe_UF<9_s;o z#Jq%mHi4f#_*+_9>2md)fPer5hkzhjLM}!^LNXvh*brP%QgP)pxo=@%G1gl}NQlp% zC?lhQRZpr(lj2uNljr^VeK`w_ghYFhpmSAuS-LI>32|uZ%i)F(GpXuDPZT<)W>!~L z1o-$s?w1e~y^D*B>1mDRFH?zaB-bW;NILccXPX(b3cBos%+4V zAqn7j1>oYio86EWG5-`nhqWK2G^Ev5A8ah;8hig+=xi|OXx&Q@&2DFEb5f^%dnOoa za|>*06D6^^fyLz!8|iHQ*;_R=78Vv`;>_TJ?aNAGp&H68oA)I~(~kgxM9AF@0>_|% z+}Fhh5Xm4|DtsO;yxW)xaJkY8+*N8+06LOK7gU!}7aMy9e7U_tydXdS@)C0a@iF9r zQ&O4@1S*DBSAYJj@9Z3N;3g>OE-LMzO?DE|L_U4Y_1gn?mM(kiD$g>~k57(?SaeJ* zP1naK*C)J9eYOh4cqG0F&!$s?zU_@2Ea!9Y3AsH!Ju;@``67>Hg+)jbLi4ckYewhB z#s*OD?oXd+$;br4;GgzQ2JjfYd2|r|4=~2Z#zcL56_fc=;-m9&a@L1?%hS^vBwu1$ zONk0TE7{CFzCfN|0vjzW`vw5w4Ba#nk{hohR+-#@T(z)puyo;&X7spG2mbxFni{u| zP-Du(IW3LQTuUj9=WmxoQR1F%b-vvlWV`#x)IVq|io=XT6U1fD4o_{E*- zva*JB>eUbS0H_PpsFISGUyjxR-)C%jN+BG?!oukKgj5X8_54{rP?3RHb>*}Y-FVs9 zT)JB?l)mChNJwb2o;kG^9GMP0fBr(VsvOUpt=fA2F*+723yZwRwZO|~uN4#qy1L@7 z*CvZ~8u9JxD=N6?_+8h-K0v~)2L{HpF81M45|YO!n+$AhibV*-iRbCDjKzxrVGMj3 zRSu-)2=B|O%A-j^s%i`j48Tjn$oZ2}V(53#Qad|wMR#^~(W%;`Jb^;398D)VxmJ5GMxGS- z*`S~x)+9kvQhXNWn-Co(FnAC^(tBgs0P29Bj;=Pn;S~$_2>={o@4;cXZ+vII4q>4! zZBO)n!6|b(>d5@@1336~LMg~+n^SQ{`lluMhz)HfM@Fb2PdKhWmXv&Z3xLmRYHG@e zUsWGUIQ`XdP+oj|Sx`^(Hn(@xw}KowmIK#@Q%(+!$v_5~(Up}62_#Nmc&IYf3e>50>8KX7-t)7uL2=TlWwFp< zH_fj909-_c=rwNR8L>G9iSDXK5W`M|jt+V$Y)XUFTJab0#w^jXu~L$d;HdI*TmZ&< z!NyiyzHiHWtiVeYJ2IjS<^`734nQ|-A|il8fp};R==b6-jx#Nm6ciMkd?85~DG{J$ z#;RUZ?4#b;-ZV8fAw<{Fk2=_jK6ny%P7_fYh)zX&*>Vq*sz~{3^cv~<2n3~BW!3lYIvUDZP{%fs1WpdRs6m>COd9Tr(tB2oq;hHMk zs_E+Tm>mo)=O~<=kv{8AMqI4Gv8TrvK7dl$7|!w(E-y~?%AIwH7#N(KoE8VSedHM4 z{RX+V#}qf>g|A2P=3mj%3otPS=M%zju7e+8?9VkaDFdttU~)abz_yox;$6!K-KLNF zrZ%u|--c>~M*)mmXLc5is>=BkVK+!ZsF+MU-;-Gk^TsT0yF{b3W{Ii@ein)=Q}zq3 zGW@}CyBQSH*toDPB^prY_t3M7goH$!73@00iaz zxH05^J0sPM7D1wpBSI2Kjgk8@Pyy3MU8@W{kz``y?_sxp6^( zY`3s*Wp&lTLZ?Dh#sByng>}D(xlEwNwt!drGZS2U`?8|K4?x0xpr}wpC7?6%#!^K8b~C8-Fm&07}=qyy^CCbHvE1 zC-Ni%@9ER0)<5g!0BbKUP61Fq01XF#DZu$!#tB2}arB(j)V#dB%r)XQw>oa&uQ|iHJ}evwoE&Y>qKI zG$p1AJ+I*dQn-@g73F-hp1L|ACDGY{H+!MhXkg2$*z?tDea4azT{DJ}~nNW7eHT@6DuZM^B zK%|?Vad}r|koDzD77%9w1G~+)V_&x;o?lI!{8 zk!xC_(xJ$>mrIC`09v@SBRG^fY(6TfsCT=$%#i-@w}9idVjbAQ=wC3Sgs*LWZ&O?U zci6~F_WR)CWF!f_4{M8>K)O^i)VS64kRN%?x6qjVKB z#DzUdt~%9xVr}MWzs#fS*RmMmKE>uWH}o#K{~s_d4{B%Jf5D{CsXRc!9Mswe_w{%# z$~S`m$wU=Upv{9oz}qL<5)7l>DaoVjxbrzXgJX@7>R#l&O@WZA{D)(90O%40|FJ1RcmipwYj+_FgGg0PP3DC| z2aq{y0Dv!eeUONVX!0GK3GO41*VQ7De;AumH7d4wuy>})b@Zs__R2zrnnwA*#cJp~ z?uaCXX=-THn%y*fDgXoUE>NTa2Ck6I*L1U+WTQK) z_@hC2B&9R+iJyS$!Qi~7`gO%=r?DJ>|Bd8m{~M}9hwN0o`?|N)>$MquN9e7^Z0vF; z^oYpoJ7#-Nj~l#2c(U+{o%%{Q^2ftpVTPk{TFCr?I5h*qWIYrN5X6Av8HU|MNq}Ok z4-gU%g(4`mkVSzi**}EbgZO`o`NI7OZ~3|f003dQAu-|7x(5OyVTrqZ`Ox)zLFAI` z5{db)!jZVrV{gY?>Bt5v`dRjxUdI|GRR*CFgb;US-LH2w?2J-nT?-IX40pBs8pZc@ zF`gWL5Yq!n;S@o4`d1-)mK49P|N0Bc{}MtGe|G=B(MbTD2^@FV$bje;7&sdMx)^+Vc|f7PRVfUwbEx+9^^HpZqy9!gA)a^&cCh6mbHcbT~lSQhD!Qr(Uq9~ZB&p355z0JjcoX*J3y{NqiSqy01iY+d~USD!p*p%{|tS4 zw5Iyms%4>X@QfIr^rZ@$CV6>J7ewoYWavLAoRM_KY@!%Y#7=!2UdjT% zTA-4@@BmeUZm*F5YT)GHtb0AGNn^MyRPPKV8E(;6f5-SN@=0a_5@6=(6;h}_cCRDC zC_uDH3#}Ip_fRqS<}*x<#6JSyt1{+q;{(2Aq-Chx-?DxLh?We}DOoiS+I*2`U*P%q zg^!yE#L;T)gudZY{}s)l5Yx{U@B_uL$4;02CD$toC{F7#B8%i7U0zp1N8)K8_# zengh~%#4+OTIsJxzBL?_2^C@>O?rK$!xdK3fUUo&A*Un2^WisuhVrZ})1JGtp&omE z&QqFYaDS_$qcOO`o0NJ+a^{`c3#RMn@Y%7?C;{6$+fnew*ah8fXy}`%qe*Q5~;DN)Wx$nl2}EyYully`e4vC!LmocQUW-omOuZiNyngS<7zo zf((kure_99yqy}!yH8eDd~Rg)NN^KXG+95&^`Tg3?amvEfc>*@OWjkN4D{q^3V*Ie ztG()RWosy_Oc!0_y&+9cy{P17SF;hKd#){~VICXnuBp@=%iDUx$ebOqnfn}Ozx`@x zq#`uPUea!J`bU)fu$?xAVmLIWY=k2=%$K?USWRstNVYg9)5iPwRda)n8}Df^6^|y# z*kLiqy8g08sW--dp!t>;lavdyKN}Fk#e+~O9hQTA8|mj8a+ukgTPq{=x7E)^%1t5S z=p>rd#+lG?c_|Cc5M0^()ce@V~d`Cz|Og3Qv%;n|_`#4Nv`t?ukT(d3R5)?phtd4=MQ6{8c{Ya^S+UM+B&`ECb3 zQ`8d#4`=%Ph-DT>mTzTbO4#6+#8$n-=8AGMD|1N}Xjy5=_{5IGjU*jZ3JsU@)6e#+ z+WOHm1#&`YY^g6C?N1D~~I-Qp&{QS{6Ok)2hB1f6w3T2*eMp*u?`I`2NZJ>M8C2U^QsX;qYk@Fc>-qGF<^EQN+Q z{=~dqGsHFI90qgr`t2DTyzDO(^2ZUC7Is)pZk+3QFdwa2yFp_#`V|nmq0x;imdW)h z2g|y}KOKu*nEeL2o=LC@480|GnR)QrB8R!Jx%{hwMu;Xpi$vLf>{eH*||uY7a2!3YW&S4Djn_oDauHjHIXL5B{nPRKO*uPqvjLCY(XoSCR! z(#^@rUH5&T2cnuqP*8hxG#(?3C&WrXdDI}}ZQ1pTyvW!3uPGH6e^xX#C%gGUUY_Q* z#98_v!%d`~E}}TsNbI3pw3ousNP#}JG8=FOI?lo^m2}&_F{E>+`OF*?hkstq<+fH zi#Lx%#LBGWsRY$TvmIdotf({^V8NO5c$_GAu&A`$`3HBbc=z|jl-BvssOnzyK*Fzr z>YB0P^1ZA}>VDbAxS~Ec-OpFl=Lh@7!D?yqCmfvAkUp$Hw}i8_%(5+Ow~4J>1_+ML zsABY6rGBP@e6O#!2w;E`+-$YqJ`Rq81u2?wxTSfvtjvCKyhbsYo-e2_v<+nKj8{iz zuMFAXfJ4M2Sl}@d8A3wH^Saq3pHF*5=;@|Wv%#`+pO^m4irvhDqP)T%7cRz@?TYNxqpCEJS+`h?b#kmiZf;G+=SCi!*&S}n6Zx%dh7N}h zdJ(Ra`5!&-D<|;KUw@yI9_)V$?7Mt(KkhK>$v2L59EIe4LOUjOD!OPjNO*dWtf;h% zT69^%tCQlqgcq@$?lET~;UED)D2_q?Oq%)nq2b3;(s;WcHN5fc4Z6>}gnN5CWStU`bK z&pgA|UiN?0)K*@-g549@R>);Z8`F+DmdcNx1i2cTh6}8 zS)+sud0UIM)jI1L5gJ%w5>EQ{z9*%Hq!!W}^mVy}g@$USZ+qo?R0Sy8FRbpT3YnGj zvGX6Wz7iDXwD4L30lu53xF852`9 z7_8#(w+xLKKOQBUwa`u8W|-C@0;?-vhXwWZUA${qMO>(mTY2s7zK}ks`WA;XGC2Fu z&{2CA76&-gIfRd_Ui|jDCQ*cROMwi6)5)Xx$Uf=V`&N?01m*7Vv@5Uj$bPo5Yr@l= zt!2W<eB zm>va4P_SgsFJ;cc!!s+TCXJS1V)9UcG~pME9d9hgX<u)`_kz)gI`e zQZ>b5rioG&E}a%A|L*3LFCoiCycyrmQDt|KQMUm`OsYXEXGxxbWFrn=k&o}NE zi*M+sG(xLN->RqriDhOuW(KeIu5)tJ}?8N4w||`gDVur{@8frEz^+_&j+dL3W9ms7o6a8A;QhBVKooGoPPrMIGY6^;mQn=s)|}4Yz#K@4IsN4?Zw&eICqUz^p7-)ujn3owJAxCwmvo^x%I(Y0CTCNpvE}uB;TSNunGIGGn<7L*yY)u2%ofX=juz&9 zz9aUaI$X;Y#BThfGF+BI5wHjI^~l&oV*Xxjf!Y9jr;aa~QBZv`$4jPI+(SWJlq8pg%xF6Kr%}}Dw@m%=6}Ub4G-kq z|6;llHGWd4ZTBCSV{jdI269(>+frmB6$02q{5`{8HeqPRoU-Z}&^=&ji~D;iP|R${ zDlroT9=->#>^4BQ7}j}v^Q;4`{V97~2`uQ*_Z{(Be={QATHjOqW4v;Q}DLzv$<@Hy{1 zAtT|j9eTeQ{qI|V@(e7E{*Be2e5YqhP>HrPQ@R@SRY>E7p{HjhD0Lkd^Yi7Kn8u-$sH13JOMbzUd4-_JrxO*Rqg zZ&3#NLQ+8~-0C-gfD1NLjVJ0O)|Nn>yOuTZ!UA`y+E69Bv>E;hkeZ?CI@{WAEG)b? z9QA^y!0DP`F*9LHZGVSu#dT&L1}J^aBN@1zo}irMFR0JfP~bQoN!r`n10X4&G>Bo= z0vW92E=xfH8j$?m*j?P)G=)xNJ2RF=H1<=SRPoBK( z{ohIKv3VL#g4&w84^s`s|3%vkj)A2quifF8C74yXhO&Jbk*P$5N2Nf_xO5EpM#}4B zJ&V)OKuK_M(Bj*4eVJEKAZME(cEu4SUXyY7vEl!(1!;^Pfb&&y~PgZ2$;0F?D^`x>z#C&E4006q)A5Tts7GgYgQfVl++5LMRpEbn@^;2Q- z^Vd2xTRA%0rRiFjTVj)NjCD+$A~iU_G$+ag#n)aeq_ueHNYc?xdFAEtem^BD%82E|ZeS!m5|eCYd30ZL!6_!*wpP2P5pSYMM@8^6p!$XB zril4rZjaTBMk9#Ie& zxhEx|+vvehPoKbc%wX3c8_r|bBTYMopDlmuyc4_p8*t{KH<^UZT;MfYyzb-RNZ-sY z(yXM0(9+Qt>(&}vR&xR}W4FD&c>K;VtJ$*3#-iQk`qK}l#?e4u3_v&@S8xfsBk6j< z$ACVqrDYKS*C0B21o)nUW&c>$J)_0XWthvWbJgh$9^*mAFg*rafD{54Q>|(M_~omw zk)`zpNR3T+KcB;1Ymiz2L3EU})r`Pp@M0p*6?NS5Gsu70Kf&zspEo^VqMc0iCsMVv z06b10Ob_+peL(U-OiZj|11R66B*hCgOFedS*#OYn#>Auv5aBE{+>-F#FaaH84|fXk zhN6_pgg&<|K7anai;1h{KD9itIlzQN!aJb`a&od1?HxFx6rg|?jEsTM;K`AR?D^gQ zih{iutt5!pD&X3fQ8IDk`0a2*mV00a>3zb?Y!j277caoY@Z3^eup*fHA~C+2oYPs` z&JF=IlF-n!Zog?;M>t!(cEKXh8(T?LOS-FkSA2 zURTjpxfXj}T?#7kEVTj>bCAcm?dh?esp2j7Rq$7Mf850h?~+t zCIg`Nyal9BKo9|F=;l;j@6G`o{7$IlidJ};dFRh5M*s(PQj>OdtsSzDA87_tZ2nu9 zMP?3b?Z_nWH(go4o8C`{u{1LL$##u{XBJ_X(p&ondHXl~?rV+!#qjf_*$Q4;&Jzx-Vc_4eR+N9f~5iG*Bie~RG#UUE3;=g-_JWJ=1)coc+$ z6ok&l2Wd{EJYKw(r~Ra#eO&%c{I9dJVx*-plUq=Lk%A7S0)}a2z!rg&ae+xT)##4R z?|Ok$QbNVvfrfcaox5=5!pPeh5Gmh*@-ZN$Dn{HslIN5>TGCn}%r_$=y3z^Y(knZP z#ZWDTP9jQSK#vo?sN>1;eM_sGMuZe5reJ!aqT(MUH!tG4fS-etQ(IemrW$q;&B#S7 zuBhjI(W5rO^NXGX78i)hglszV)q29{dyV;J9DvyMs@5U<&DD#S&%T+m4FX+gAh)Hc zvYPjw9q`sx zWep7t6_xVp*vS=G4z6efys7A`avw?fi_5T{=R7htNLTBuN@|QU~5nLEsjH`9qzNO zvuVBpeoVBhk_qfJ*Q8bgI=DQUh7Zm;EMRY~#d+2%RZM zR{Nr!o*+&sUKz#tG_DW7cWBsxB=J z1R6pC(>b{OBNz`M5z2-s3gfGc0*qPIO0@{D^(=ZVOXK?*ZR?@`kG;1Ht8(qz2Qd&7 zK@gGdk_PD(q*GczK%_$ukY<4*(%sVC-HjmK(j|-T?pmz5Pcg@>0F@35vwYYDV9`d0GgD>d#T=~OJ!<>_1omH%=>W1me1>3v;JuwSnFqo7LQn=; zT4&Pl{tDiWg5_T?1HG${x&I#{s5_e3ya28Sbj(0^=dm`o!Sef94vcGE7^4(-vs4$sjeQbU}N`@g8M; zeejc1FtT#$edBlJzJmL;e}gI~Nu&|~6I9vV{PKZy@A2jY@H=EA_>KUVo`Ui<|W=^5yMgaiw@9y@eK%iRTTJ z9AUR53khRivGTEhU@J0T&DdIxt*~HaO}(I-uR}zE-MOC@2^iz?oNmb_C0(Cwj41Iv z?-TC?+#PvxOFsz%s37>o@{;N|)m2;dC+s`$nJm803FZ<)Vf!hF(3b@wR)B?*kLAnG zP7U^-fz;g(@V^>|`WJ|qSm`IB86+Vm-eSWmO+g`K@fM{A`cuQrDj-i# zR8%|M-PorkA>~qUeO59_BmXCp9~Ctqdnn^%bwcU65-StuiQR7gLVKX9it0V;8TbmR za7WJ?R0hD;8?Rhlh?oFE{J9&^;u!nKa}dh3gI4;^wib3@V(KV3Uge|N(!IRA0Lxj< z&Q4#yWi5;v1P9sn{LUxpM?;guQ4HlJ6-4Y#_vhZFkdYm4O)yA2L$+F zwlrMaH6Xtb#TZw}0x|nHEV7Sx{)<$!yQE%0iRhQ?=D!F@8IgY=%UQSG8y@1=3pG0W zJ#jx>eIVu&UflG*0FIX)Q!x~Lc^Mh9XPj;+4jS30v6XpR^-_RQHW;1_77dD4XSDzVrdn~ z0A6a^Z<-q!h~{&$Jq4GS;y6_yn6IJkoLiXb#@psbU(RQ7sRpz|(i)uA$a@cW4E>LZ zF94yU;1VKPWDTM!#sZ7s5)kL$z>5i*CmK)e=TN~azZOx@3j0e$0rC6L&{PS;iv*yk zmi*OA1N;N*cP@|MbPNn`KaOevl=7$qfVM%uUNJQ6+qS3g@V)YlU{sFU*GhNU#w};e z{RS5C<{SuEtix++cu4j8rit0u*+bf@62( z9hV2X8s`b|V@8egH`OBU`UHn;Y;1Oy6`JDWUg!Y3R$|;)0;o$N?nV?L1rkc40G-SM zn2FxXw>eeMF|z0^pKGmIN8vc*Jv@8_|c zLd^Fl50Hqj!B?ZB9HBAGub=lU#1rYUb2Mk?V|dE$ZM>NDisd?@SHxm>_=-U-+}1yJ z4P5_6C5W7myc;9q9Z;t-b?cgL(3)0oq zRbN2Turcg&0Q~QAKDX@?P{52eY82~6Sng?Sej z2q-L2Qc_s}sQ`Y4-o6>v&V8(UK4<%P?;Q3nt9Pl^P1m)xXTU6egpZk@le50TnRIM9 zXV{Z)(27l7Y59Wy2c6_)Hjo(D+yv;rI*ZJs?0TV+r z*^cm`a0Vzepud4O=>mkWt(@f3ZTI4btM*MLXt6dJ6z*5AA2s^VOH0YTs<&=Ev^2U; z#B&Hp6x4x)6IBLg&YDw`G@2Hy-=ji$zjqnFIQm!NiFu?>uGIhnelZ|-^OVe`zW@p< za4xVrE}S-zS>+<~WLhB^zJGF(ujkqMKpghXLL`|uzSgmR%blG>Cj~+1l}GSbGx*Nn zW`BYrnW37c2dT7pCXf~A?(9t9a_lmO7X!f@r?6-dWB@A!Ni^mf@y;8x*Z`ZKdr=Qd zm^Y;AdupK(=yxfJhM(UdX#2Svae!GTvf0o4Xb*pKqV*`s)(oJw);89dG-`TA`}%SV z`>%1TL&ItThoY|uO0F^XZHs>@iZERi(U@BsPcm7|C-P6c7mcyxzsW|e!=WO=3NZ7z zhqbQ{j+8D7X1D~PnzK%hC;3m9bjsgM0LCa_KvIL6$tVLzIQtk29zvrc%4Go&7UTfR z%F4*90zpCK9^(^JVQz_Qv$NwN8s7l?x8CJ@ek=;9{>E|uAUgnrd=N1|PnDY>XNtgR8vl?la-T{EC4ZhheYGMtLZf*YO_VR zt;rDpZU?AdrO+_@?Qt<=kh#dz14MSM{@i6&2PmeIF-BeR%#EC}8a;Z1#V`elK?E3E zn|GV1`%8X+G;#fzPnbZNjvunV2oOU)Px4nCku zlARknb#Th1$y>`QDG4Tn)A_kdL7`ji%>Kf{K?KW3uS0%1> zIT~HJa&|TURq%q{TEgK-u_!J)x?^Pz;F=N5`2Iugq7f{xksPe@*MEsbj0FZj=!Vt3 z%$c{{4Je1bV9UJ~$h|`31VO`WHa|;7>RVG+<*SBo=*eN8&rMx9 zewQp=UQ^*W6{*oxuL{8Q^!mnaz1mOibw%OvqOt0+?HyH_e;U9-qsYGokdq{hNAg+~ z!%Is`E1zk2Aly?$PtSEDZ&yIFZ!!JSdUp2hQ<=rSI+DoQD8`o}H$N<%Rxtj7_TeH^x=_Th%I$L- zuPl$$03V;euCA1t%0JaJ^4*3ErQ+-!>9}4w19<$LeB>QZPoR^yb3jp!!qLJ%gD>(T zAi&Br?5c<+fG2!&j!KTdw2;i|4uKV!`R>YF1O+KC|vVyBl(F!7=ON*~K`_)FoAvr;3|#v?uf zi(wpiU_1}x@^o%L-4Do|l~?WnS{^Pjke})NyAsFX)U^^vVTil|){lI9P}O%8q}|_3 z9cPr&0)?K-^(DXQ=~r{N0jq3IR~P&NOcaeu-39wnL+Is>*4_WfW02>*kHc0~R`z~~ zX=5RQpMjQf`M?3BmDefU>nZ=k_rl(Ou~R@W$W)EJTQw>B`Mn@DS!9J5%6YKI-CE45 zH+vb!&1WwCzKTJ_F{S6DLS*mH-=@LuRVHprE$?JObK>Tn0Kj=rR&k$7n3|lpyYdWn zsiGz?U(}XSFjW|w!OPCP+<)CI|Js0jxPs=ywt@(XvqmjH0A847?HGc7i{fNzxHq>s$6T~zOmEQyvKKyte+W>V zXj6#a;Y@`TQx7D;>v}_#o$*@x`n2Cs2LgpG!20ASb6qZ!PlarRc=NQs1*;>wlv-9T z4L(Xv($^UUh%_Q~4P93oMhDav0LyP^SXEV3VX1QvW??MhozQT#ACa5))*PI0AXu&C z;%3mUsk3Z!eK=hU*qk43+JR?;>sz&C?_gv07lsBWG5;jqP4vh!fx`AbkXq@ zA1TE@7yHt58$}DVvQ7tiFPa#bcKOeAydJHSO8(R!k$>Fw^RRx<3v>IA~q z7x(l$(9yhr&Q*aGGzKVijZ{sznw$54t`j1YPNQ5E_VJoi$3F{+SYB-$zvXqE+S*!J z*jgZeS#5Vj+gyOPx)B@-q=pg_X3Ap^Cj8u#%x*vMHk?N6mddikHs`Qh7ESWW@jMTt zibP7~GSK(`NOhxIXQoSrb;N7W*kB@X@Eg~vR*i%1Zg4UXCgFE-R2230N5|kxh{)k`nY)sO{tyoS_wgsf&t!Wrs z9E*s@At^0Qge&ZhnnEh!4k{5xtP7YNhQB9=3GJ8Om1i8rc1t9BOFN|*VET$ zLAke?pmmb>>M@%QpSO{*^*IXlkRYxTR9f zpku|LSts$1Pp9VHToCL6RPn1{Sx8CkPmbii(R*WRo^W}*<-ay$F@x-q7^SAJezu+H zb~WA@)hitcxOdxYVeLSnXU)V!uuwhz^xQW7 z!Zb(emfn^BwZ>S$J3I2S6~vEO-pIldfID+%r**Q;6Ww~j2|=&x+?(5UB-4u4@uGqO z|5q>{x_AmC;_$9>Ygg3x>W>%)tKX3$1FTwP;X%ViREQcm~)ARMjT zYN;t?YWchlatnglgWmcjfG3pkdSl4+^TxD)n~C;TIs z3)+ATLMSDm*mRfL|DxD%o=U43QBw)){v4nK*L5gP;5c65YQjH>uPJ61UMf5kBC( zv-}H#R097~j^$M5e?h7EPnDK&MMXs($K}mo6DXT4$?r~t?Kd{@q0*gM^q)AS*$gy~ zCP2jmC5)B6xW61N);&$oUW=-$JNArQflAmTEG%~0KRgJ)CmS5FwM3lglOY zDZG5yApoEhDe!SCfrbV#Pcg7RV9cF>uD#BJlNZd52w;8Vu6n_Z9B%Biuk5*7%y1p9 z2nht)U*mUJ4<i)aRhFuAwDkt&dgcv>m?BFyn6K`%Rq^>6i7C3dz8BE?9gNW zYk87S=-HXmc_rw-D0VNjxnTk9(|WCazzZ>932f?2(r=zNT%a4A8+-8p1v_7(I1ase zW$)�@*XCn23m@SWZdhH{^PHKsV8-KkhN3dtBTrB&2pg@bkuH{xt|6lsh-skP_v= zMs?NTKq|1UE((AH2DPo-k*Gv`i{o?8hnH|b(%aX^NJG=PF`6eSFF!Oiv@?_Hl@c0C zg4A{!ph~!FtN0u*PruIagVzAnOTbt?#^>s2&x8hi-Dqyg%FT9mbS%`lP%@V(1N_xO zD8jnC9}Q*$na8;OyYX_PonwYrz~>Az$EEpK7)v<%LkIrXpm77#F`V6?vXt8^Q;Fbr zTY@30_&7kWEh972@6m4}+B5R731=HnLuORx5~O{al9lD%z0@G*V0+brizq7EZKtYy z)55^OQB<^RY7rpJzzu?K1jP0(UU0ra501VyG&>&v2e>mVJdS2^TMp8An5`r=wHMNnxJ}8#J zuOQ0$B07n#AYLBE1LCq^4AL^LR-`b2f`qUHX)W~o@Abxp^`$=M6Z$-0TDY`G(3zP5 zv6yvge2-ud!7BW_D5ZNZE@pJh7l7k@RqdME=4yNN-Yky>aP9q(qg?KO>5}-H6eThx z@;{R8bBeJ6{T&%&@ml8z{0#7QZjKPZTrLXLYYY6s!hne2?ukv40g6ZPKWC*M-ckAA zU8g8~VYB~)npd-Ka5z8~_b%y@!IHayK#=$quP(+f^~H$pN;h~)bF;Jk&gi&8Q+tpH zIw0wFM*SvqJ+sz7HVwS-=?-|17Uu?#Y2Y8y+6w9uy93Dp2ALO<2O3;_^z?l6n?UYz zgTwsNVstZ>^QJmg$DbGtKSE<=)sZ^#Y-XZDKqpGddXoNhFx5hYM|VDrqoyVSQU~g@ zO-t5hX+T7ahJgV*W-NB$UpM}tjiOY|23`0c6#aHIxI_4f_R$02r=N26Ac8gXXff z?pDoHR#K|PdZf&I14yD05gQpnlW%p{o=yS{Ia|q&kOn$cE`s+rXr8^j}ZwAo5OuwsJJ}F9RpO7zyE)L;B`!@!NJ-Jul4XM7zrkAX2I3A z0B&gi*6i#+$KBsAi@E#j07gL7A8LRB4ebY2PPYBqP-IdthGt{%H+n-?wQECw-fbE8 zhZRRJQUR;sE**tzF)b+s|D{r)Dx zznhBH|9=c@_&<(weqYr8E132tb$jyq<}VA>wncwMK_3F$%k0rv3}1_DZ5{Z)W$LjQ zw;=2v{621Pg}_!k1h~%0viDB54NM&zjiHkpl`U=X%UP3!_TOuLAmv(lO{U%!ygdl( z6||zw*V;OQFG1O!no{9QHJ-B^2Y^BU`uX~-W{BVuz3g35{FDU_$M*-!ELCAKG$TLC zUtIq}L=XTue!oX=Y#S%I`11jyre-I(<9C_`4PL(I-7QIseDl;rlbD<_BSX$g`> z^r*zUES-Mb_|2XX&D@S|O*0c0XsfG`8q)FIMeexLPv?2>H_TJh0mCM9Ov*qCX@lta&fH1!a<`+Dtxy-kum z?u?;W{L%Y_m&cWRO6bCvACbpjo>(E zt17M^E$AW)7zwGzH3Lj69p^{zg`U2}N@vQOlB>re3XMt6$YP%9%ZX_Ra^$?W5zIf_ z-_JrTkx73999jhK=EDlyoTUM8N&Y|pP~fC!pIDEM{n#0TL`PKTCU1Ys;}a?YB#j{p z(c2kmnkLzaxZ_j(%_6c2#Z@JW!LI@50etmm8>08zr`k>&K)C|5$UnH3@%QwK5v0-X zW$73M%ne#H{#f*fhw^vMPE-lQeNDC1RgU~Fg8+e#Hpj6Ou@w+=C-+VfFK&~+os3jb z38;5Hk+g7iw6)c+>O4t)0JwgT-T;qp%qnK}tJ;TWEpGeWbTe2H=H9Ny8A%dkh=>11 z;)CCQ_mAv6vMujoR=t@VU&7DLBjo9=? zkE0$6YtUU~PU2b$b-naBfB2t|4=5Zs#Ng%7(4v(wGsJOgJJeS@ZP<%a4(|i{&_@~* zhzu&BZFwfw_~z=?Ib`)ySmDgQhpbdtVJQUSrv|20J|0~ykKv5~#*?4(kY36DY7zZA zaMl=px^MPFvuw}o_jdnm9{3-SUS2%X1uO7`UmCrReoGj?i?Po=a}k!nB zzX}Mq{q>j?t2vRpfKw7G8wY? zlGiGxKYyON1Ae<#m@9tjX#SOygC^x+S>U4kJmA6e2iynypn&Q-ZV!hqxC$p7kGL*) z($Aq#C1xtF^QR_R5ob?gD^krT?`8UQ^%xMZ*EY{rB8qBfc>07jCXK!%;*sx#^lX|} z78`L2b!kDS?T)O-JqVlgz-qy{t%S`ytdP^e{dG--OkPDOPEjhmg{%@^J1#AV!G8oPE-$ca^A6ZDT}_? zOtPh<0;3$Wxfcqr1}btZ3C396H2Fuw)F~zO!}bTTt#t&*=ke4C}xz$Qw7UlZ7(Eog#hW%Qiecv z$M?`fRS;FoWxyw%&UaUVq)BCxGnoSSag`Bp6j|NfclILp}cm~{-3GrI1-A{?7 zIU9kCWCE1uyWV`4orP<*{La0Zr}q0qz|Hkt-*9v9TPUR`oZr>nW0vW1hPBeMe_D)z z_?P>eMx@${_jm%`75&Djw`?pVo2HOHhP02VJih8TO-{sq_G^cV$oe z2e&hPQA{AQoyv7cuAT=i+WhrUd4#8&kYShKRAz!U?~_ZIYm;XPYhBUqbx6U1sIS!j2v}&Y`+trnMdU@=F?|F>`EQvPI zSLy_QsoM)t>NUrf>B&lQQD^5ft*fk4Ik^H_F#R8TMbMN_z!w$_&?euZCfXc@IP|_? zT}|EUt!|vvgG@o3iN$^@r1P0RiOYmG2fnAY3SlQ_@ZRdQy zhj-_#mKgl?OnRlSSgdB;!BqMBx=p|Kl54ZLHCshK@ok1_B2`QzqCVTe;l@%0e zq`LwkDDUB>)kBS(@>?AH7r@r8ld1M7;g`+Q}tlZ}rB>b@2%`aR|?+;@|K%GzGZ;q?!(;7A{HPWVU{gFvcyYkQb zCij+`j_z!xG_iOO-|NMdrQ{UG1%AR}dG!NrDYuRk|8C3b5jJr`XtpHatpQACQw`qO z>aL-4VA|0iSoXD8BDO7r^fI3JL(R4qWf{zHHMvm}CAs3f$p!B|HPN#;X=RE6UW$rL zwmp-zfm+dt=d?8YF}n?T9ApZ3)2QRGCsd3BvAbr8v(~-+=#$g8c(})~#W)Q~4FSp3Oa$1BMG3`kJ$jPP(*~87hyutMD3;kDOi32 zD5tCoOP_Ll_c`8fxh!8U&+^XAtuYKI%JV+>Qjn9Lc)8nFFQ}#({&>d`O>bkMqw;u<$0nN}@V&O3 z)E@p)K|9)V%1F7e(9KYZ?M9nZM$ktH9UBAqOSV2BXT0pJs249UUJbtBU|c*oBW9h! z8|ONTt?kaac>P>+CC>v#2zZLjfEziHH!rG5$nKxk0uMsH=B&QHpRc$zS`gOMaVM5w z{P6hP8+y;boqU1uQMhtX&0(io)H}VYL*OV`&e8UmvB5oe<_W~mT-74~jA!#r&*%8h z6Fb9`IKYJWg}MS&oTs748zoV;A0K+>WPqGM=+im;j1thp9_{odj}_?Q+th-Y{ZQDP zU)+$g#hVR)$%@|}51NPNaLIe(&|r91%rrs0+)CPO>)Y1r6(HI;4u{DHN)W z=)E^F?@7e8`olHkd{7%Iw&^;T<&`=?FM?fbVLhj7deNaNCU2}5lydpQ;J3dViWJ3N za3u7vWUpNb-jKyA?nw+O)BNapn*a5Bhj#B@`Mj+;xKi7j3J!+`ms3)Py&qu%^L764 zmDk2X$Y(27z~I}248ur-OyC&y>G0pU7r-KAL%=@r%k)DGn)C!Js115`E0;7IrNby zjwvuq>5a=mqzF&~etBHDF{*Vlrmc@1Lv2t2fYLh*q08ZSj;3 z8k1KgefwHNTAP$@6D?Q_( z2y{h?_vr|L&)2pFmX7H!yI;(P@DCZjFPHnhW$C;N`|KVyV&MddN7l|pK5nQc-d%S6 z`l+$vzP8o0UYAA_TYG!nqo3#0gMN$D&a@`-ho2kp(OJ-p$f^r|PVIr$*Os{8k}G^{ zm%F_{pyxa`7*tKxYlV*1KA7$%YSomWkWnj+qX-Y|e@E@{(%0HKa6bj4pS+bHhYv73 zC_QylhuF6T-3qm91AS?8Xz_S52%XPe;Ia8D5*YLP&AA1%uwW&1)x2SiA6<}OKlsf# zd!@wb2sPzxUNbMSRhGmUMkPb<5eP-f>rjWeChCT$$d z1kWvC=pH?mAzW`JJ*Yv2y3>#@`{|(a)1xGV!Y@N0A#OwQe4<0^}@yh2O>Vv%A;1JvQ?Gm#qaEc7{ z&DTJ2omblzU_BUl{^?i?vH~m6v&c1_M-1ahOf#=_nq@hIhA6B<-jK&V={9k-;Sc31VWe)?iMALsTg z<0X=c(`W8%#4nwO@%))Yu44JzT-nbZbp=QlMRz50w?CQlc}_Xp$DW~BSh^J=oW8g0 z`l0yKMV9Q1MD2q!E6zYW0%rB#S@o}pU>;W35RdmkNFrvpzTt08-zjp8LLYSOIMh}e zyQl5cKfTOvgHK9vwal{K=kA}NCAhkJ5m6m>;=!-rfi933h2vTcDa7a*>xE;>@(+bc z(4B%|1YXHa((7t+Oz!fZF9i}IK+y!d622c-;o37wvc8l|6E7vWj9$EZy*eFsW>6CjT*i+q8f#B(O6#oAGDeya^Hr z2izLVPV5c1#m33cJ?@A<@2{->8;8K7x`K{@9;n4l3>x1M9R z(I!r$b8?j!1&*_&&u1WysI>2uMdoKx%wkKpL*UPMY$If7ZEGM3P7(0t;nOff;5qC^ z=UMMeps1pt8k%T(aYFp`^YW)vXoR=#yt^M=tG*B2|8C*SSbd| zZu~=L9;&aIuG+q5j$LIv(PjVCmeA7m(v4D0&6+8n=!kFCVXd7&+txr=&a~$&UTDf_ zSmO9{v~#s|iq03MD*f|a`+glQUb#~g4%EmqPE6a1)Zg283N!W5J|4XCq%g{*qX3-t zAX~6?fplyWd$2mf`@30DTy#9+Xgd3I3Km@@q_1zw`h76YkpiX-r}65XjE2U~$BFQz z1vsr6uj3@)?!k0}Z2OZh+SOIme7S~h^hSeR)zuG8TT_sfa#>ZW{if`lA3pu9!6Ru<=X zYxCJB$qm56RGgk4HI~>q$Sq2mt}_=HFX6|Boq|Y`{M{ zI$sFvRRKlh#=ysXitj@4@ln6?=y4>@htD>RAWxsHMtv<~qQAp&p{VNRx`1DV9r>;Z z!DV?o$V3q@1m~9VzOGIiTk{uDMUYU(L1?6>T4D^GOzg32sCU5I^PKNTI`pxL_}eZ`++??&iDEPOPLJic&|)XbCjq8`AstkY8Q?{CJMve#LUAGC>^~TGnrKu z?f@59%9w_-lKS=ys1&F+61y4^r`4tGgO?WQVLuc;XBrpO-GQ7R#@~^PhtTBqDM|!^ zM1NCIm|fB%An!|5@o^JbU-7e-1Zd`ghzhklY|#=(=3$G_OY%pmD|<5~z5~{JuXg}9 z6Fl7DJBVur`Dr=ZPtTF0IRzP>cNgGWo*m7;C#`^A_ge z_Qn!rMq!L~r2Jjh6Gr2y5luPm5H&5D>{H4>a%!l-sy&DehZe|Vc_M4rNty>Be!x|n z%A=iC#t(0(|um`!=a^Vvie9r zqpk|FzE=LpMcek9dooSxL6)PIrz;&bQ{qEs-phd;5Fof$isjYn&kw|FpiPv{w&sVq zSN6o$c@^;3>UFa@iegn-`Dtw6WVDJ-&#sL?2dETx2(u1?#Ea$fCB$ z<>hiWlJ&l?Qc%lMgQJ~eZ#3NU+nkI5y5^<8!lfdHtGyQVyZ&hU16c%-hg-(~t?)qV z@hv2_Xy-;{+U%hV@m#)`MOBJ{x$WWx0;*yOtgBmioPqv-uGPob$I!AMRLoS3JRjsY0cpUXhB z!DG19&v>W@uYG_i4umzrkiDKGn@o*NnYBfJKn*iX!@XR%>t`Xrvf>67Kg&}s=!%F0nE{rS)Tk&+TEbn0H4)16e1ijpR zgWxaV2EWn`tmHH~%LCB?|7K&G0_=@sP)K3*4@(Y5`d|M8CA};4T;JdJ_b^d}$ z(`23JRBoVj-9>keq-hzpvcqn_pAlK^pAYtY3Ie&7kVFw$cP>0q3BZ~%modXQqNn08 zc%-z_1bvcg@=%zsabiH~xwxpTw5V+NRySp(T~AI($r1-91eelw|Dn-~l%U|f&AgNX zSSyoTuIU=4@FNvv2%vKXHqa>U`hyu^@{0-#Tn&#!vmkHIgA-ax{DDFGfF2*O^yMKc z0tpS+gE}DiYAOnb>*Gpff>?9!7Ke{|U%{}!B`j0$V_lw2ro;xt)d@>&f_Y~hcXU3-6&z?SjcR}(+S1~J@n}se zUDecX<_h4tD^9XF9urza+dw=B*K5}7lHm9Sy$$rwVnL_PPpt= z<0GF(x}R9?Oc=EpE{(<%eq(HVHLe~FlgkgV;yt+$fn_kf>>{j~I0JdSYt+__q=L2d zc5?Ep=d8EMz8TaU)-NvJe(N&cUc@TXGnDFKVAs-16*&RHr$ z8ebLEMnl99VLWt{JQ714*1P+9pfCmUQLAp>2@%S{ra|TQn->K$%-|Y$4llZLE@n^h z#k1JK4W8Dm-{}b$bp&%pHO?-r%yz$_daKs$%_mSThb(VDHHY>N_U>%KHavo(D&f|I zwF7--GN*GR%5QiJzk_hNDG^QP{8q(ts1=wsX|19nM6fkj?QInXEo}Jb>nRF#UhJOA zyi0mdiq=r;e6dlws`#ywkYR}iz8(s;j-x205E>aOvhAB5fWpQ-FrrT_;t4vTVsaN5 z2$GnOQgEs6v>M}-TVt_G*Ms{6G3^8h)atFSHy-BmjLA({$NDt*o6BkY8$Bjl?)=OF z_86NL(jk@d!!Y8Rs*{y2-$7yLonoS}F}9=>V~I4nL1m2J&|`W(l0F|FZ+k-w%!LVc z_e@I!gu9`mVX9sCpwJoq{=O-AE|k2?pMLY)f0V7v_?bY#6$)r)#MrXv%{L&~Y3rT3 z%zPS(McmP-Ia7&@GUN*aj4n5^c>%uLYG8|@7X94C(N9)9OSF7%z$02Hhw{ZNAV8WPZj^Rej%D1u*m zpo$UegES4S)QLjk>w$)A39E=sfTJ{sar^PKb;^IKPSt0H3`FgT?q-kaVppe&g6(p9 zQ%eMyWn;5b;J{FF6kEwd^)l8?^2~>r*Uz-dtIV)1C##KP1$ja!!l8 z%sevywf4b8afpQ5H!7o8?QglXT?LlW;Rnm=Ahl4iIFIK?0hhI`RhM;pBw`wrv`==l zml~zB3Kl`8$XD+)`@QuCSt}Ej2~wx)!JQxzhLnWXmX~6C>HM|r3=C4vkHvo0_A1Mm z50;**$kyX^YP<)`Uj;>KCaXJ?=YRq!9Bsd?rKN7=JXY5nHw+RLYPI7==ZCcabDsf{ z11l{eX^w`vEq|Oi?rEoK5iPsjCRP}cj13?t;XY0rx7lCEZpsusVKC{?!aBY54Q<|R z5f5VAlJoOR8ttTe62t2VE>2u4;)G>ER!G%1Jr@cp6GgodYOo6l%qHo#m>YYw-$Vlg zJ{OQ1f>WwhOHf(I{a2yPG*5Ymy`UhcM>Y^UuhWT88X1I|x?i5TV{qU=f;BuJdjz`? zc_9}_W%YJJUbKHyI-(~8MetxZw~XYhL5*RF)$S*kFT?7>vFp}p*|jioSa|n}Uss<( zRO2zUHBVF^v3~E0%69dVXAn;7TD-gCy<{x&^=bo~(XgJ~hJ2&C}5_BRe!OPrAV|__RXtg#|lIxz{glWYVRk z%k$r=Y4=0svBSzf={GulYA<`N(EsKQed=jeW4c;o)V;`DTB)Uoa5DH|<`Y-f zf*TH$whwSqnJ=^)_uAn|VTUc#l5cEuMTD;@+2f1y&}))e3G#eg^tIpV%`mhUu(lMt zX^NZN<>~#6jP52h+(4j^@Es!X2rhB210zNNqta35Rc+6Mc_%VG)~|yco)8^#zmSj0 zI$-##fM-=)@7bWPE0^=qO8Rhy0Poz8gWszSkqARTcJxJD6r=FFqkiNf*^h2*>$jt5K34y9sC&UKsbn95T|9cAoLWl_OsL&IyVd`X190BJJs5#qdr*``Z zQi*C|vO04+iZ)p{|2P1@HjH;#)3AHD8$Ym8D{FsD?}BSB)}!Su?=Dm_@d{5*!!OH- z6>x-`$Bmjw-6+P-)C}r81OafXdgD^NfU=;a2uD|Z0M{w zNk?7D-$Ikp$A0+TVEn#tM;P6O@u0Grm02NuUpI4!&G*vxkEV5-1~TYK(^jF4;w)52 zKOLPMKRXGs75M~XRF(m)XsuyImmVonkFu?G*zRTpkDvzkssBnO7Sc{!*xtOWsMU?c zew$6f;(q&Bv&=jugnN3psOE9E?dJ!|S3#+R@uo{v)z8(i<>IZjT78U4#t%aLrc zn7Fh|(6|_PxbKI;XZ{27PHL0E!bIWLvC7K#7AgcE<{~R6Lc0)+(en4R?-o7aA7p+o z_O-eRnGrtUAbPI-IRTSTK_aQ)Rs{*`m)$e*4Exl?4-5bIs^cUcbOU${iLr4@P)M{P z;{u*|9j7D4coOjtj%14e{Ttr6$>zYGdpxl&=H7z1cS_xRera(7cG;%q&rPSbKEtXs zbkuYw0#SkQr5<3@-RveCc6{?j{*o0XHFYR{!`W|dlRkS0P%Um>*9^dhBmjjQ0z(6L zYdT@?k{q!yCd0}>Vb?Yiijvsy4Y?(t#T3i%M1+$$9WQOQ@MqwWmwZr9+O;$L9Yg8P*kr7-j_<-Dg zx}7PXNl5ot5FGczSKiNw^*b+mdek1EPTC*EpVS6c=evjcW`v{<72&?EbwaV8RZ)v@ z?Z~F5Q#H-ESQ6-Hb^9oBqli8g0=3%WZ+u~zo}o9|&|evgOuCaGar!auXe7A`$uB$z z(dRTeLD8eH> z1rhY0K=0c8x~&+wV$YXgu-w-JkE~{Ow)Yitj-5#myqak>pon;;o(mg5UE$%qpsg4R z3y&OO-ANwl)X>llp(O3phxL5V6M7;vn3vc&FLQii*mKi{4T6X1z8Ip5;4bonV?^ZB zTvKlKu3)f_b^ zlG}nM$lg2O)!%zrB7A7W7#Sf59elcMq1W@H4V^Kndr9UT$PT9El#soWh_W`upQ1Hg zZK!oiP*UlPjZO>+4SSewx%LV&UGHdn1XGn#y&8#Vt5cynZW5jP$XXUq8#qPS5gUEl z66AE7C+v2(d*1M+d?%r~Rt7OdT9%uwHB>8@&cS0u6nZgb(6-FNyQAnv^kuZ1g2_)Z zxCstaat~Bkqc98^$sO)q+V{++4_8`Wo!lW~CC&%MCig? zDiPmrWaiapj!Ha+?@9#~OFPL=z0g*S-X}|1yM_L0Jr?QKWBA3!d)4_Vys_=aZk;lY z7y}cX)J*&=baS#>?8BAc#E=p=wz&t&`k99WzU5@f2RTYXUKhUv_i4uWk&9zdF@U;m zz#ds<;C~L`(UHl0X9}Tx31WnU+iWcvG3(z>W+~UnQFL5_`Yd5+nh&CHw!li9CH;FR&Sd8o0n)Nj8_`rbT|c6EalL8jCIb{{LnY~LX5F(U9F zO{g{rNd_P3m>7C#-`A6~VNj_v_^>pLOn)&VG-1Y~g}u@piKHs)!%%HtMSuNm~oHc*ai?lv$E~JK}r>GeegPJ}OMynU}dJTkgJT0|rv& zZm_uL4K^NOtDKB;W<00bLHGVBwFfYCB4u?|jks7VOL+67IHf1d6e9asSIL~M zVPi#8)&)gqSUm}Ou7eDxLEI>W2*e_IAJg*+$k*TcUmVSpqw4)%?0scam0i~^Dk_K| z4U*Cg(nv~3hk$fRHyi0jVGBx2cXvuRNOyN`>F#C|XZbwO`+n~k<9uhFGsgLO82q{i z?6~i>=Dg-L=ejPL6MM17wCK)G)A*hKqA9K*DNn4?`O@*CH3{09$*3-_`tolNB6NKl zVo2(0T`^&#WlxIvC-eL-@@5(dPDO-2raSH(SmWj+e0cB-rR4!mgNR=?Hh9gIYX9Xa4gd z7MAJUMYiU;FX^*u67#WycsQN09W=e5K9<3*R=#{Pg!)T@lZJ~V+b27g4;!bEw#Sbz zE(l6IikDGI}!_8v3k@<;)kRr`ey0uJbLa&(mcYaAaT{&VhP#a~@Iz%>rHzwZBM9uUFcVcWm` zm8kzWmtV+EX0iVkya)oqEctU_zd(T-t(ty!zKEEBFi!F3@;C0hb#QF) zZK0TW!`be}=)B^es|Xc}|D~edjwqv{lT&L5KK|Zjk1{c@L!p4@>TnCN4IVe8Gv6v* z>2!B<-P+m)B4+|#r-#EWkG_8W+RzTpaslOsor#=KKRzE-2OIk}3{mei z0sXZIHL}yeRbzp5bxDe|Rr8v2xj2-bo_@@)J6c8lGP>QXN`Xbm z;JqP|KRbPjMoI?lPrh0>eF-}w{XMW|yT5oOWOs=k)23gB+|+^e=WD-0`Y~T&pl@Dz z5h}Ekz-JW(@elDMm-}gTl)hK*#BGSs{V)R2AWVX_yY;7V6F8`UHef6%BC@=}G)F{4 zoF_CfKJgTd#9%h7FVx=Tq^Ye$vwpDDAM32v(;v;SGsF;M@{7;~O3l``85db3(&K)g zii*=gy$c5}uI#(A?8OB z85zs2eLYuif|ri*oTkM2_~k!o*GI*o*8ofem5kcw%F5U>zBlRJFZjNC$D6#Mvi=(WV2cA-XfT1q|q1M+>^#T>z^ZhF~sg6kVF`&+PeOo<-g@uL4_#pWSPFZ#6 z0dQf)Wr@9CcjSRAe`%8l3YMEJ(#5&HR21cIX~6$$UlTl4vq$)xI_tdoih;)YHrhLQ zD_%2x{3)}4x#>kQF#6oRT^Z{B*8PH1NtfR{IVp;&9e(Q(M~)mDJqIARXM=1s+kPA0;>l2?NED zpbTW6zYY|oelfML7zRuGYt^6C?IQ^Zi;ldm_D6nRLx_7_Q{UKFTH5nVSah95`(|g5 z*!c#6^2t7UTFjRmTM@Sgs&%X2Lz?i)7Ua6MULUq-!)^7 z(Q;>2`1I*0BP>jUj10IYU%Bp7k*@a_9j!&1U_GCxM%I7V&HW&Fw~!GW&@{xtT9jj0 zkB%Nc?KG+^)kvi?O8MdogaN!}%HAyRCA0#4EPr;h?y$#!(c|(OiW=tnSVcq z&7~P4gkRr^lNa-vhZ^0k4CdBT?CtFfhJa}8CJ^R7c)GxMaS<1qmj_PSCGk3+6Z}3o zUORDr@BoTVOuPV2sx8zwo$1KR3JW*oX~a|#a^)6SUY=g$-jylE|JS-wx-Wm@@dQm9 zoJ7>t-Ywpn8xzcSVy#v$wWtJ^>L6?=hQ3;Be9yHi{JQen!7pJTcB}OoVq?^<^+4vc z6xqRId9`wmJU9>=8X6iA6H&gmc-wV_!Z};1)ogOMBJsktCl{~R>wfR_nmla=@xQ0J{WMUBwJlS+iBigltMyoaCg4^ zrf6cKgGR*LFLO>ynR#ugib&={CggURT2RDvz-!MOacWP^Eff54{ zm^YPAn!g$A^joj^MDi5zcE zVa~ojAhm;MC01ATL|+~0zfyhg^ekEziTsxqGUnpjcs_K5qp3&%p^8s2C9wI%jf?e= zhnqYXyEynnqxA}U*G5Gbj7T(KAao#WLJuf2n!oct+_5p`HeC#T^5h97W|3gIby!m4 zqXQYmU$;+z2L61Vr@wXeKB~?OnA2#Gk&zMq^#!cTaB6LbyF|PB3VtUfaD8D~s*bAp zy((ldwshtE{@~+B+wPgb7>j-tB_#q*lifw3d*?Mj82Fcq-MNF=p`z~;Rs`wAW)G$y z`uerjU4uJbKzI?A-}ds`(62A3n%dIR7RTe}IYUX!#I^_BM-je$&&`LIi*iY{c&-+Scb*%%->(Z!c>er3K}F4HrnD^}WI+C!Clf=5`vc9Bj_F+ubVUTFE22jT zsn+2Z#i;jH?i-MLXG{@rgT?B|Bxp=Ux|biHP%Jhr+47JtsEV>OLc70JC*Nwv)8|G9 zb{K#Y-8|>EcDA{mknK}d&CWKthZ_o=MBS9xtNEi5 z0|?=|eTtDGesot+R#jS7#R{+Q(|dY1WrX1aR7m$>7t%n?&zpcSfl;?kTUhuAh%G#y z`uXt)6GE!DZ4aOTh*Ti)vg$QZjT=Us-H^X)u0P`ZHD?NhEE^tB<4t*iSX7|K@p~5c zftCnXL|7O;r-_yJZasHNTLNFSs_gTC(h&{xNsTc4NHPS3*OYg7JZ4p6jE3c~1~iDd zA6Y8Q(-&?z z{E>l;gA-Rm$xd;yd}V)PavPSMtm*V6RtolMh~}Z z!)v#;x02xSgLSk_M=W0ht8+#Ucd&(rjl>)|{R1tV9SOOMG-_L?+l|IIS)m=z+;1+u zy>}UCX+JiF?>n7~`lB8g=%?l9$0sK{%?%y&W(78k1Z2=C9C;~#a5s~l)t3td?9c{> zhMaeoDjeZxi}^0iL^g{RM_$7M+vNcPy{9kP-Ou)m((~gZk~~>n|6cvR5`}{^VaQ{9 zi9}{}-hYribXk0l0AcpGzX-Vpk`=2)#*q0Y`{ilHjbwq!N=L873SlV?*Q0P_jTw!d zslvg_(>&HJ)xLdD}MYUjPm? zhZzK!L!^g?7CZ;)P@mr4tlZR=i}d)Qf8t40*xKT&g6zx`#FT!g3+y0goa|vY4GetN zoN{D+7U_Pz2Z1%UO^;A8Fwk$#Nu6c!!`&w|f}mA-heeN@_OIcvvx|+}D@w-GnXw#_ zTlgQ9A3QvE?Vf3g^wG)!o9ZqBf%#Y-#A}CK#E4LVx2hQ85WP z&ATYDun6ch0>xX9@c#%M0l%MWF5vMirJQ}XJtiS4O2NR+&T)DK-os_Bt$}Q!t9`s8 zv&QAVsZkrvAN?4}6g4ZPBnn(~*(~g}ijbb@G?WA~kgZOSw2cqKX(?B)D+_?&mDBFX zO3N)=_MzFi+iD;r<{i?=6nE^&%8kuJ9S$}wiIAN-&^cQ#oEf9`j<{ZW1VfZ1z>MJ8 za^`lk*(U)fv$;$3gzPq1EP6pSAaJmb|IL*4J@_TJpnw1q zud?Zox2#-NfLhoij_P7P&Zb+N^~>+Dn_9?0Rv;cy^KFupOt#RwG$6OAkjOF9G6KBd znFN^jgb#0ld@oQvQntTxlIG};RFIb5Ze1$?lDB=hEX#66rRt`GI^BbVcb^6ssUHlJ zF5uhv=;Kn$$Mf5r;!8>O%~!`bBj6k|euE&$*>0wB>?3AixB_4hU{2RrR$W@Z9+3-E>wZhLjSSl)am@X{3;AlzRuHckXek@LMdRTIvgjf(m4^6hWc7Tt^eC^86AtB1w zZ2UM59AS$z-goahnwX0P5|e~LfHpDaw4suXjERXVodH0Tur5`SqOhb3y&;{7lG1Cj z%zCz(o!4QLmWrte)Ihd}VR}w`(bpTem?GWzl{?}=1X%?I3=|Y&O``s+hF!raJW@3; zS20Cu<_WO)9QXT;G_aJFMY>ONhFp}Bg*>H^SXo)E#V7}qLVbX5*KFm6GmWXK#+x@x zOhrK1MN3nIUEx(bNybQFVIeq1#msg01Z$}ZQL((-`86kdz3n~`WRr}Z97xWUIZ3N< zasrqJB%sMNA2~R(d&hFxw6?>%;ZX!DulmoOgmu{&~=bA(=y9G@RejE{#$M{{wo z#q&GH+YjlEYf~p-%CU|X6c&2mAE#}E&v6-utkd|N_n}A7*XR2=w49ZCT`s`{~*8=e@)ryD&T~bS8Hg*q8K?V7OfFss9?zTb-=5lTy z$L)?wW^@s<2QqE0_xRz}iK#-Kkgu=LG*D(^K2SnaU?~~w>_k2~67sm%`p~kfoHI&qWE|1j{=Dmg_-@iB=oQjlf0>zqmPQ;Ld!- z&6+p~*rQ2GN)~F?N9$-__f0a*Yr@(!AA*1dD(_Ka24HrBwY-x13IaO@p7h7Z=j&8E z2b`bL?cnQEFpteSeUP>g2G}k48%7K}_ZMq#uO}GBv76w0U$d!B1m2NB z8MPb4q@?I*U)y$$(>-cz%$ARL+5G*bjE##k29hJD7v!;k@;ovuEVcMH4Oq$B^CUjK z@Fl=T7#ImYpvE_8X({cBPT@vR@*)) zf7UN2Sid!{C!M#c?x0kq>*;a#T6l+4>TLV5#!6oP{Fo{bWDq=D+~}B?u8yX9m;FRg zqvi~DihLhfT>ODv2AB=#7cx=qH z$wkRv`TlPDU~i?~x=UV0rhrN4J~AbSrtBl3Ie2&g>pu`~0UjD>VBy*EfQ*@DvgUP1 zgnK;=44HsB4`;_%-_BmbR9+rx%~aAbE=5E(`A4P?T?VBfa8nWWg&sgbK&;5g-uYe=9TO82TdjJH%L}`|UL(Y3 z5W+wRZQaxI68`c9+%50|;}ijeJi{ckm=j{Ef! zMFSp3g3QcqexV#(tiZWySs*fbY2_NMugT)VTuE8U-*Vr-hmb?fEO*8wdrZnjNrm$5 z!MdgT;A4>@^U}*4BYJw;8|4iRjXyg1)GzWnud{#%dj%TP4SVkt^ZAu7g%rLaxH-@z z{5f0l{CC1Xu(SYGIVEiNY1q?G~kzZ zde`z^TU1y|R`O%{%sOuL3!j5IK3=dlPi^f20)jRTY?5PvB%T3vMlL62j8}b+qgyic z?MrI(ccKRSeKFbq?A|sXNXdRBN z*kFXMKqwxf@25%p!V2mvdVr7`jI}pGw7woK{6#NC%-G7HH9V|)u($X}c4m#baA^qN zbI&Jez_{?rxNrSiHRM@fKn>AgDw!xQqoxwYOMm5eNapP^C;y^6BS7{k8PU{K8netS z%*DR4-`ZwCe4u!NU}<4~lpQSX>} zUoJx^B3S?uax;l&Sb)3&T(V{!*-&CEMF7-$LeG$0u`%s#Hh-j4+rae)=lSj%eKACMf)zH8Kj1Z&Vsz2R~-OQ!YPg^R{b=tA1Hrpz5W*wQC zx#xNZK*(P?ApDOn-6sb-Ue_D-qy@A#`S02IR{?>Z7p$C>DG6PG^}nvG6GTN}$a7l2 zZ9_nuL2P4;P0&J|zj9NQGBq>=YmqUHME51)t9;QHdUD=A4-kgMzn$0le(`-$Tk8&7 zEAc`1rr8#QjPhop4;VpuN;^C3>YJ!`8S)j-GXUOMMp`Bc8wXxfBb&@t#f z4#1q*dn*<_K^iC{Fu|B^{~#|vSfzFG66UFZEt>?h*s1l@+hy#)z&^R6^D==ai;a)C}bLrnRo{F!fruqVRB~Ukmk> zSR8dp=p)=wl{?%a;7)euMt}kZ><3_LFb*+A7YUCDx5%7t*o*n~JIox$b*Uloa0-cc-VPPOlSuk15&dM1T5uvQ2 zvIo4a%5Vz!uf08|lS4xGS@@hUwst@krlO!QG`0r9z4@uB?^l)?g$Xdpj3(k8*Ku(H z?FIUf+5kqI0yi({3L;Z(`BeKz=a9Qr;#e zEnn}afF3rj9_lz_7NNREt-78e)uU^jk7 zMoGWC6P9A@z5qTelMp04CPuH;gz4-F>c7$hKoB7TK|g=~)UI`7svLWXj~_(BSLGkU z9`zZL(_W2FciQz8)MdEQ(T_Yl02zMp_{oQq4oU=SGp?4`D2q)^`^m}Ao;~vu7eB!R zCu|x#m@C*>sULjXA8u0%{Afe0+qALpE{iVg1E+(3creR7CB~ct5Uz^hZXV)0uFiBOPisHa6{Z)m%FDB)m?E z=t?RoKmtD)bT*GL)ivg1?&x@X39AJq6J=vw-n>9vP0U)IH_q!<|X{OE(rKCW= zp6`!c7+P3pTMe)Ti+y$y5rcg=eSQi$7TvpVCTN;m?sh?y*{@b+)sep)>ht0`=t9k( zpUhRR;E8lkf5Q4OF>Y1w*t~V0a#9YEiICUn`)?T1l0T8iqEw>#DM~^hp(C3CO`!i= zQc#dk@1IaFadz4exT?WuaPa&6O0Dz7!-p0PmBL|uVo0AhN7@#~A}&nL&9{n)`S6(& zB4b^qV6eV9_o1fShVK$+#71}O!O_a0naT2@kQhh6cYg7G%6D$d>vatgL5gXs^%m-0 zXp3<6z#@dR@m z3=Rodw&Rf>6i{Q-5kYc2=h%g7*aP=aJG-VkDD$ZZI`!oqY(&t;0rK4t&F11kh^IEQGvF^F zum)zsa?7DCMSv+CMjkd3PJ!?Tm_v3pw!uW++N+#k&`|g}u7>0TH|`sP==!UD_>c*# zCC2>Djy`>A(CDS7x121xKiwL;^WIki?7f!PwUmg6!NBzys+*;!4Ep&cD;edc?#Cp1 z!Ug$nn+d(3$uyY!CMh=dJrG(3ePd>7lc9YbOVwhtHzjArBx&FC=*ZgdqxEzOmO`aM zM`{9!^Yeiz*HxPDQxNz706jT50R%d%)Y|gnuc=Qd|7|M@zc?jLZf8ndyn)HE{N&`A zWWHHw!#@u7I;7EkDnywk1URdxs(#4K_L@G~*+nqo>IAt1etLPaRz#BY#w9g#Nmm}

H@QX+s59MD47n0_fW*KfNCqU)*DRd2 z+oWtN_BZdO|?K_F@=RhMF%wz~Qc;AC3`1Pw`_WJQ&3pY$6MwH9WEqXd zb^E@rtsRvx(6*T@)IOqQ{D8glG*^FYovG37BJOwqdfz069v=cKfHh0*)>e_JBAdej zw0`&JA1ZP=}CGfdX%>F)=Ycb+p~jw~29Hem@B{F`8@SBU|m(^Zssjo4c7!w(W*J z3&EwXe^BPkNysw1xWtK{$}{l?1-3TmZtHdA)%kx2NJK2%xk+ zLlAg^@D@1ip_wf_ae*`Hl!jk>EZxkkqh<%YV`GlO8dJWhh6LEKkrzp{IC8OW3H^p` z7y!!#Vm`ZA*M}UGiYvA}5!6AJ`?Q(@thAjbLq6{xz6}NFLHheS=ugX02(bWz)<>Zt zBA~yA&h&x9>h>xJ7`4vN&x1UZ+ue;L6Vu>%QYVazM1YCFf#?RR1yzX;L!iU{_m7#u ze@^!NUo|bo`iR%nf0{{sv_(3D9F498OAZ?qTGu^!;(v)oG#(VruTN$vS(KB?jFcX_uRlT^>^fZjSm`tP?`vOxR19uxP&6?nV3mU~Gm zC|K^R|NW&kSx0mS(sD|KK8+eZm5>0X>1jp9)0T^&ihhGzp(gLrzRga9rN4@{KFO(yA zuz2FGHfN(JbTIAJcVKcB1;YPqoo5$_JD_U{I$n$N$|sJO1;lezURTFoW5o4f9|uEE z>K%7f->bXabnAnTSzNh^Lc)#Qw*QM(dVlIy=QwiB$iC-$rb9`lb4BjJ3}>Xxo0A-c zA)47nd(m|>fgHv^YV=-L_pWn~5BLRuQ5t0}t&I&i!pO)dJQj`UsH94-f$boL6vUvv zN66lO6XWUEAI}?uxL+{P(J6Wp5WJL2=4GOK^9IZwkcnw(x^0xW^AZv&+dKZn@6V46 z+pg4Aipr*e2p9U4(V?; zf7G!+O|d*OGSX@9N=HT}5IvoWrrKrZxa$S!vrl7kqPq<7URWT?G~YETh>H56lhe>t zJ6!et9PSe7(Tvbn@@ta_QBqX}r-&1p-vcBBq|r6r8PA@+1k*#~R-YdI^^nKOTlGV} z2F3ty>&K6uB)@SyR_>$$G&>+OdV8nGwg1f{^HoD+;Flxwtp->56CeyQ6&0puq$9%W zst?{imFSvUS^_;W3MwiM(5*O*%v)@FBp?8X!22x7#qzQN3nT?3Wek2j)ORrKt&{G|=YVCND$x-* zfop&sc&+XJJjl9Ld(-BVS%I7OPyXI#2JQpw#LZTH-QBc|j13M;H+_?;-#z-m&@u0}jio*v6~wS0kqssL{V}K& z7XyomTW|dDt?P&m2DpLFJ2Y%VAq9Ai@M4*nTY`B&N@~i(>~=8;43UaTM=0i7H~?O2 ztHrS65>_2$nFxJoD<&af8$2;00*pIF#N^zKkBszOp3(`igvPvL=9(<^kC0r!D6V=x za+)waIr^u^`ZM)2x4`*A{q`e7CQ=w1IMFsmS~{}}h&2H8<6q1E{ekW@=Mz+G#43Ez z!jZ&uK*{v@hwk zlY{39k_SbmEsqKmozC1m9)W;~oqcS)Usgpf*jZSjz08`CqN2dq zS3WQjbU%{Dv$u~A;>fr@?yi3R_c)6tisCJPLPP{egFryP*6vJ`B10bJ5itLe(G{|{ zv?@#Ga&-)3!bV1oJE^nn?Ch!*>V_`&-GK<{27$+QNOa>nT=3I9nMP^R8jx4%nV1?4CM+I3dIZ`O`fU=^ zGkhEd0twZ(nD|6&dvh<-W!{tOt2nG!#BoT0?Xe61KH!c6W0cNoBQI6^8p$uYj1yH-xpDTS9{X0 z&~Yz#C$M{8zvHYaP*3;CusQKVY10e-dkf`L_Hp<8Y; zXy4L&GwJI765TTV4OEsEXQ4)u8p?8$@o5bu{Ac8rJ#og%7Z>s&zoFOm3v6< zHAjPcx3)I`XMYW>VeRIIAX6cFv5bR1=_X?&sEz+#O2hQu4kxcyGcz^VP3MoAZ{N`5 zfO@tKe&J1gsi(eaXki2HFEC@evC$qH7%2MVDA~Yh2K@0QS zmIa~pzaO}I4%S@lJOHze09^zPP`9f7H|SLIkE>uc6|YF%X6FQ(IXDjk6Z6ttY$e?g z@IJ2qUA4rBa!WXiA_fL3V=J|i9!cDv(G>9Ww&0hND&8Op%UfDia3XYjoom|k?GC}# z-`fts47vSA1%nCpzs6B2U<}J{PQZ~Pq~XY*uf-eQyfQ@ zWwj09>_8v{j3k=(Sy`>`KSS4MQU5VqEdqu7d-}^v8vox&iuH^BzYJf{vHt(%zi#mV z$vsgTAV%Ow?s^hhY-p7`^dlzy;pCI?n9*PI-u-#34|X0>u!wVgdfxAg*d(SFDq<=$ zuxM-!Tk(}&rCwu`V%c^w81FK>Y|-b&yX=4k>h7DCQZ(4dUJbiZ@}D!c>g^uKIduj*L~z9TYaC9D4_F~UXlK) z@$h!#f+e(xv6m53>A?+QT3odT9w|GqWm0;7dhq$h0( z&}ISBme*hQYKlW(Z=`!A#dQOo%@KCyEhBNee4a#7v-ju-H=20<=k%SVfs5Blb>2Nl^3!Re(2O97XLaqJrKcxTG*jzOb zL#lm!W-!~wVS}TFZJEm=d3wVyH;k<%W@(D?M&(V0ptxV-^OAw4WTNo?gpBl(^oe(Y zI`2)!rzAzBeUz^b@EnZ=){yZ!8O&(kd*>786%ghT);q2TV;{=>3ae;v;4=(%>LiG6 zV=VD-rlKIvz|fF=F5Mt_A-=ao@gn&BL8g3p>W27>f+f{))8Ih@5<>pW)hs?bE1FW~onD7)bi~^0zGVBPf*1Dq z_0HMhXIfjcX%g^@G*XvxqHI1|Gg?YSO6WdnpTdCJxGKlbG42yq!{k)A=Z=0|May?4 zM%LFyBl{A&K5UE9s~?P# zHh2i@2^q}jvAyVJ>rzg4OEz}-x!UB0S#*8d9N-P(Hx%HcA#0Q2V*=ZqBzj zH5A#X>Ac~ry&1phmQh^FX)U;?7S{`HsBh0QqH-VH%gZggJeuj^E7*_rU`~4x>I${9 zuxrFS<#O6#EMeA*RXiO8LV!Zaswy9jCNoTOCm~-g;Us%~&w5D?kc&#AW-m&$Ld*7> zdroe+H^y|cPEfSpPyhma77t;$y zyA_D+TRLGV$cx!Zg+0+t@RaM*?z0IX)-MhB!6dbMk*3KeUfFIViZ|KgiAtR7yU8OTP#73e7zk~`ZR39+8nxTo-`x+c-G&%I(cWS3gw>su( zN^-`n;JaX`G_N#8XwWK^jp^v1V{M>?`vg4UfZL3LIB|9^q@0RT1c`OqaAT8Y0goq4G$W#O;4ya|9axcu@*%3@A1I-ceKIemccWbNQvJ6l zs3Wr_mBl=!hN|N)aYcN~hD{YA`M?U^t=}Q2mZa_Qi>qo|k?eKC`xGXBjeKV%A?_po zn6@>$njd%+?gIx|6Ym)KxDupOXN6T@^7m^aF`N(a&o_qUi<5E_qIc$>!4S`uw2w26l~``q7;v zo#PwKkLt~0%@SYUI-m8O(FN0XQuBBO#{bqpm#*%wrmJGN<~|?C-)@P3hJn~Cu)*UU za<9E|R1v(0CwWW!u&rb3NT3}{pW`fEoe}ryVy>pJtniC(%gz{tekO`E#d~*m@y+Ij zH#{P}pFA2-MqaMCX~XzXZumXxlp*i=uGz;NmUewKH%(kkXjr7gfvd6-N!GO7ez9@! zJ5>8&%5TktLLURwPA6hV8a(;BA*bBpF7(2~atbrXXNt;= zDhnDmxm%o%wuzb?X$P2s>j)Oo%U)sQWouL3^Xt~1Yqy-x*6hX>?wr`1>Wo(J;3vdL z=n3A!%9oz{n`>DB!4|NNAS5tb~1(X>5 z)aP~gQyhe(2Hsn5-jRX8*hkwNi5G)HlGgFjetNqgfK*Qspse{=&7GjJ%M<>*Ht%pGvDhNCnGBJJ^6h+xf}+ohLGo<^R??5Vah+) zPgv_KSI^wbC)l01<&k7xLB(`K?2>n^ z-t;fG02@{kEwoZ_4XI=U!`9m>{$F7$Kyv3tQ(_!88;m@Cd@B6PCS6$KaCTHL%lU{^ zc|?s)@_tC=OC49;xHkD=%MWswcZBC$4UPRmbUacTYOcRJ?-vegY;4QBr_Z~M=1n28FE)x=mHhPCPaZ<{s+6QC>jPU*Sdef&V8{YkD zlPX+C9K}XzstTfG1?+U>@n~P9w6b7`Fj0OCapjz!lvc=)zhOLNW_n{M)=kA#g{q#7=I&$zI;66u;+l6=A`fM6o(Z~I)?C;0 zYvlDy@zPm|9pBY3L%v*Bv?Xm-0=ZuZ>h%gr>7N;i6P?v8tv39j@JFGF8kQ$XAX^~w zP)(wP(085psv4?sFXOI`HyNbNrR3NLo>)_@YxdVC9P|eEm8X}Jv%D*%)X3jc`HHFE zr3*a9KXZNoYZ8YFCr?3{bk7T@>+DYakp4JbU}&tdwlMMFvH9a&pE9*By+-ewrerzJ z@V^%0k&Y0|Vp$8RzdW@t$K!tE_q0~&F29qorZ9j5t_LUh>qmQ6-@wJSjtmD*;VBRH>ttV(Nz~qk6vs{+UHG_H1q3z>S@x)GcV}?$Nr!% z&J}~RjtT`|-_3=WOl(TrAAQt1eXn)%@wL-66kUf<#|GzOY_5}E^{2d#n!{>Vt}{0* z3NOlA{*tI>iR3D$xv2P4$26ipL=lT0#mBkNBOyIJB)loft8*@BuJ2Ze#sjQ1FR82J zo8iE1g``(LXWP&!QFNlemz#6)O2ccn7N72`u>;w1B9opq+qUytc|3Dee>U-V%@*IodFs(L5Vzz|VY6g7pBekg|7oDvX7 zON-~{rw7=6hav`2wHQcAzt5>xvQv`9Y!JM?ZvP$8qD7kP_#wf?+$XtcKahe2&6TJQ z0QF9u)B%gC1l952@vPO%b^a6`!@BSg%(E|jON(8DRkw})EL7+z3bJOcR&@wpzx6B#>=llv+3tc+a=suX&L(UABOS207Y z>@$W*mx)@W>Za2~a6J}ey6$U_l0(kMaZT2_yc|MzS~4P81qCMk7F2Z;LS;%gBkPT4lNbtv_h2ET?4 zw-*b=b#yP3byK{Oa3U5USv#o$zR$)`0PMU%KTo?Gc=)Xx_aYldD$kqc_4nE%W;&i) zU;C#;%$i(=R{WFhh{Z30>M-TXT^wa`)mY!No^ZL_pUa(PeC+EH2~-gXeYkRb2RD5f zLv@t0K~s5Wk=~clfsS%u{m6o)E!;|-Oijo+$7aU5TQv7_N<;@`!dvqK_O2ExgOqpo zfl>cfax^u2`WyLjQ8VjAd*MS1)3n2g*Ui&vEaw+#2PmV@3jH*w7Zl3yEqVq`X+F58 zLfDsaL?TY($YG?9=U(nm&x2E%iE(&8UNLEeA=RqP2UWkU({ZHAq@2AlyWjoAKK8&Q z&0WN?&~fpK)vbnFG(C5778d<=jtuGSaDgAOSEQ}pm|E_~fhZeP>-%amu}|&?zIK$w zadSU>aM!~WBUznxv{xEiL1uVteqVb~n4eD1z{|kEyR&izDD=$5QnujleHd`EVZLh)61`vceE|JgJ!eOsEGUB%l*?a52I`-~J zs)#W+er&9b1^@_QC@P5v?0p7~3HG-0)EL*)V8GS2Whf}2EUu>zQ=V1ySAs0)>fE-i zx%EL$)YzPgOe}AEa#v~IyI&+42*939Bz@A5a9R4d?EzujBVm&5K=i4}qLIL9)h&mGf0IxZTTDYh4= zN-#zCt!@arnAhE_rTkT)O|D zx}17PG@-M)K_F>p3`NJebz?DKO8A@%02E+y`YJk^7LjIHXFB&g{XCyj3`)FL&V33q zY3^H3_^j2j))?zY)xvhpAtr&yqKrTtFzzCzS`J6QPDWp@?NRR2meJI$^9()r#66>i9-*e zNT1!;HU~5XBnA{NVDl~d?$C|Pbyuy_SFxMZWKbG%&i9_C_;&&x?C*!&bG-#1Dj5<3 zO4}V#-D@}DlB>q;L`(MB>_7I9oz zs=-&FqRE<69Po;7RMevoM#&;*BKCP@9d$saTcMs=p1@k4Y85(g(5!hUe#^)Eo0bfE zftkLD;&ygjig)(JlI{(ifL7xJttaRr$M6+7yTPT z1o#G@UxmcCx`#Tic1S9-L@+n$)LlB|HAxa2#Ngrrd^B>{Jf!aCd_r8kEtIi#X5-Vc zyg=;~5gy^@6}z8jG7$H;6X2}cqJ8eAR>bWgUuJKd4Qwm`@SUA?mU?A6-P5k;PA~ug zhb2#P)$ukWwE;T9Ue^KicCbjpKK}mrdjFtQ?|wa;+Y~k(QoTm?Q9U#r%YH@Y0#di5 zZtiXGN)9m)dHe9-6>k$X|O)CQ@MZ5Wg$)MO9*g1`|V3)>vcw&-4l{-Lv zZ_Zb2xo15w6&B0g(Yf@84r#AO7fX8F3u=faK2H~K6LDk34RQXCnU0>89#A#vDK&j) zhG~5xXcpqS7?jlPzoVUXGkQPB@wLDD8-99$gbCA^&YY=~8X_B-Dft*vU9vyew1#m{ zD@^xoBSh)YH{)j}zx2_}KInq*aq*oy5k)+{GD$P%C3@hyn)lbr5ZR1=$o+VB79G#>}!;cW2KXF!*W!j=eZ1@ zuxIHsLo4jA((}Dj zA=OYf6Q;L>l*?KiwX5q7n?r|$CcXHc00g-;(lGYuj&wkaOKEgtfhH=9#k&Qpl5=MiBY0LRtI{q!Wg{^Bg4E(3IXAVW>4 zLN9eQax4)Z9C!2kbv~w(pZJh+sJ!&TSpbx?Y8t9nZjhoGHqxc14!9ml;y13p=LG3fIwyRPk_@d%HLG{vO9!P!O)Jq8oJ~ zI0RW@S`1va$Sp~F4G9QtbH4TX6F>D$s7XYo;kbBQ056d88K&0$|v#m*o;<}$f@u8LP1mqSsz(%aTQKsDMU zR1%IO4O+UMOw=<#z!Dj?zelR}N{H^~jjvm`U1v7$PM5 zZun&alma)(TQN%VR5^#r`k*9e?n-1Dyt}q1fFV9tDPypYuxnFj6nwM$VJztQ176MD z;+RkhsLl%WbFag1sLYh#2PepZDiP7TK}}KV&idD>V4xdJd-H-SYfCr#$#DA5$cddM zQQh4c`Jaw@D^6Q;egC7k>x^nLOT(x*0z*e+iHaab1_5DQLIez801*MDNl`~sh7KVb z1R|ZGj6n#}2_-lQ5SmC6V-%^<0z@G6BBHc_2tsIra|yFMyL)zj?4H^A&i#?^vkJoflxD zYEweF+G(2H8XL1Ka33BnliX-QU_w-Gl167mh591m+dZvw zP>wmYi!1oFX^(~4XEoM4`yBfsiv-)HJ<@7ocXPhc6fkB^yb<%j>NQ@(4k>A{3TPBwamc$&ChkpeOPyyNe-8sWwuQMBhlLs zyCQ1FL1U^iq}S1|>r`)P+ARZ{LULC9zCqpmAneC~%S=!&U%S3rUEFq3y&WuJJ_27j zkuUzE9Abl|?kTW@;gyN&j1%mdQ;M*mlv~S#;Gf^vzkEZ4mh~W6r_Hozl&_bK7)TaR zK#YhD{Olg{H{zuFc{SKtnUj#^?;(NcK5*2Tm&PF`*_Pc_Ush+wGcNlWX(g-JQeA?? zkmvj6Zvs+|SB7^NuV;HzMcU9X1#gxfbf}JJnzhipCCB{AuPuq{quI+PUj#`ty;YJk z6Ucg}=#QKV;2?xpZriMzz640Jqb6vuPss}5onLds?mf<1FOBz*`4&Xi%WdKsfJr;P zX+r4$lp9kskC^70**YeQOz=uLCnecz)Zb3dVBL{mvh!Eqqmw2}{nPf{jX$p!qR1%i zwoGBs(s7x1s-j^JLp$OxY&G(hCByWIq&>+a8-uE)=Hq}ZPKC7vV>6X%ubTm9O05788q*_Yxn<8js{E=e{ zECoN$!&y!RPP3?V;S`-VaMF$~PxnqMcb#a>A4q+A5QGDc5AODDsmxs11>(oL>k&jT ztqrB;i7PUfYz^|tt~xabp=en|Bc2ken+|=0D$6c2bs0k1a_?K*npV=s-C?yY{;}BZ zL}1@E{G%pVT=iM?Z;OOlzHL>Eyh=TZadG}e*_e8s=ldsJM=-9&0WSXuA^!Y|rDtaJ z@M?q7s^!}=jr}(X0o4P+g+BdZ!cdQfh}X@rSN1V4o;;_Ff0_SXURKU0B`41EG_#j4 zV5$1>b4hFE45uE4%JwthMW*ng7aAg4HwO2(;vPqTT*Zu!qz4=T9&DC#DZBk53cNtTYu_EoM69R1}99C0UyUAD4v|2U} z7?CaJys##?k5EEZP!K+A@9WRFKSUn-#vutZL7C4<1O_zRn_rei6EiV>8XevmurFTT z?rNjVi_aIM36BP@(*j;jESD1jwt7-!&SBtz-{|YL#xO{;t_=5> z{-P!CovGMb66Y4sU<%NOayiSN0+=tc>{jWsLyt=zpYp^R2F0Vq+$$8zKO{Kh`DyUT zoZ{=;_TKXI97llk8F2h*TPF~3`IXIap>WA?;|WpZ@93HAG3ep!;v-&(0K@RL2Tscm znvCnTB5GUwmYIuM5)R@bNV0f#s-n%mVmHg}2^YMx89ycT#vZ{s>)D>|U&g|>YdEXU zQ8WGw$!vRr6x%V%8blr-tgu~rwCf}f=qkRb6rjk4kh$kE^k^T$n&!jln$CCM019Ee zWJDGnaHBiX`8b9pD-SiBKV$V0!(BrI231Hza@d$Rt}MznPEj0@7`n9E6^9GqlOcas zVDn7m#PoO3XU>FX*m*llB+k4shgi3h_Z~XUkP$9vLSAhm0r$9mA#lyO^avWZkE?wD zb2W{5R250fE!C~$m`!-9em7B#RM;BTx@kRR=$C3`W%~L9nif;C(D&BNUI6S7VE+N- z=0%OQ(e?PXzIlXT*@kJil2y}9BXin|?F#Aryv+ddaL!8KgMp zn+;tEE>{FAd_fcr!A`@9bGsCC!>Ysf4I*2jwy@mtQ9PUdhjs)jxj8m}Cz$hm>N~q; ze^H-qfkTN#DEcday7xC%!~?b3Md+k*r7kQGc6f)v%armxaOJDpZC0UyhtZtKzDNL2 z``_)Uz;zOOZwm)jRKvgJ^g?JI^AMLbH)yr3VA=aks>;1xz%16>vU-4PLIHbZeZ(hM zXubnPjjKV|%S1Vu;-`5ir(aLfj{@od!k)1S2nS=lu|7u#JShVhMw6Gu zAc38JH<*17GO~(Xo$gh$_b@(V0t$so^e541#DmnycGMl8Uh&5-v^P8#XJ0kP3oo+$ z>t712q!j0Bsh6m~u_giPGNy*y@dYhXPMrGe6hqSY8nqV;<-l*jJ@;QM__p z#o+9fnh%o6v=~;tQvpCYEMl#MG2_Dp=!0O{>mAd-sp z8%J;?(S(VlOrqDL(F?t7!MXmu4ggX7(ET2wN`J$>yLVLKi>0$V-TbV@qqp{FSDQb9 zyjsnH%m23%vm4#T$EB)0lauIHRt()M2lr9r3^py# t=l^N;Vt;1f$!YQ*WQ+Dce|P=YU5=*>xW}h@#<#!6Q2(OdW8{s{UjfdzpcnuE From 4c1d91d96f36da7e6ab9cf09d1c923d33e52a529 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 11 May 2026 22:20:16 +0530 Subject: [PATCH 71/85] fix(anthropic): inject dummy tool without modify_params (#27620) Anthropic rejects tool_use/tool_result when tools is omitted. Always map and attach the dummy tool in transform_request so CLIs work without litellm.modify_params. - Add unit test for transform_request dummy tool with modify_params off - Adjust parallel function calling integration expectations: Bedrock Converse still requires modify_params for this path Co-authored-by: Cursor --- litellm/llms/anthropic/chat/transformation.py | 17 +--- tests/local_testing/test_function_calling.py | 91 +++++++++++-------- .../test_anthropic_chat_transformation.py | 47 ++++++++++ 3 files changed, 103 insertions(+), 52 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 2f11a3fccb5..1ce80207552 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1809,9 +1809,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): Translate messages to anthropic format. """ ## VALIDATE REQUEST - """ - Anthropic doesn't support tool calling without `tools=` param specified. - """ + """Anthropic requires ``tools`` when messages include tool blocks; LiteLLM injects a dummy tool if omitted (no ``modify_params`` needed).""" from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_messages_pt, ) @@ -1821,16 +1819,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): and messages is not None and has_tool_call_blocks(messages) ): - if litellm.modify_params: - optional_params["tools"], _ = self._map_tools( - add_dummy_tool(custom_llm_provider="anthropic") - ) - else: - raise litellm.UnsupportedParamsError( - message="Anthropic doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.", - model="", - llm_provider="anthropic", - ) + optional_params["tools"], _ = self._map_tools( + add_dummy_tool(custom_llm_provider="anthropic") + ) # Drop thinking param if thinking is enabled but thinking_blocks are missing # This prevents the error: "Expected thinking or redacted_thinking, but found tool_use" diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py index 6fd253ee294..3c7e004b62e 100644 --- a/tests/local_testing/test_function_calling.py +++ b/tests/local_testing/test_function_calling.py @@ -268,51 +268,63 @@ def test_aaparallel_function_call_with_anthropic_thinking(model): from litellm.types.utils import ChatCompletionMessageToolCall, Function, Message +_PARALLEL_TOOL_HISTORY_MESSAGES = [ + { + "role": "user", + "content": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses", + }, + Message( + content="Here are the current weather conditions for San Francisco, Tokyo, and Paris:", + role="assistant", + tool_calls=[ + ChatCompletionMessageToolCall( + index=1, + function=Function( + arguments='{"location": "San Francisco, CA", "unit": "fahrenheit"}', + name="get_current_weather", + ), + id="tooluse_Jj98qn6xQlOP_PiQr-w9iA", + type="function", + ) + ], + function_call=None, + ), + { + "tool_call_id": "tooluse_Jj98qn6xQlOP_PiQr-w9iA", + "role": "tool", + "name": "get_current_weather", + "content": '{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}', + }, +] + + @pytest.mark.parametrize( - "model, provider", + "model, messages, expect_unsupported_params_error", [ + # Bedrock Converse still requires modify_params to inject the dummy tool. ( "anthropic.claude-3-sonnet-20240229-v1:0", - "bedrock", + _PARALLEL_TOOL_HISTORY_MESSAGES, + True, ), - ("claude-haiku-4-5-20251001", "anthropic"), - ], -) -@pytest.mark.parametrize( - "messages, expected_error_msg", - [ + # Anthropic Messages API: dummy tool is injected without modify_params. ( + "claude-haiku-4-5-20251001", + _PARALLEL_TOOL_HISTORY_MESSAGES, + False, + ), + ( + "anthropic.claude-3-sonnet-20240229-v1:0", [ { "role": "user", "content": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses", - }, - Message( - content="Here are the current weather conditions for San Francisco, Tokyo, and Paris:", - role="assistant", - tool_calls=[ - ChatCompletionMessageToolCall( - index=1, - function=Function( - arguments='{"location": "San Francisco, CA", "unit": "fahrenheit"}', - name="get_current_weather", - ), - id="tooluse_Jj98qn6xQlOP_PiQr-w9iA", - type="function", - ) - ], - function_call=None, - ), - { - "tool_call_id": "tooluse_Jj98qn6xQlOP_PiQr-w9iA", - "role": "tool", - "name": "get_current_weather", - "content": '{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}', - }, + } ], - True, + False, ), ( + "claude-haiku-4-5-20251001", [ { "role": "user", @@ -324,25 +336,26 @@ from litellm.types.utils import ChatCompletionMessageToolCall, Function, Message ], ) def test_parallel_function_call_anthropic_error_msg( - model, provider, messages, expected_error_msg + model, messages, expect_unsupported_params_error ): """ - Anthropic doesn't support tool calling without `tools=` param specified. + Tool history without an explicit ``tools`` param: - Ensure this error is thrown when `tools=` param is not specified. But tool call requests are made. + - Bedrock **Converse** still raises ``UnsupportedParamsError`` unless + ``litellm.modify_params`` is enabled (dummy tool is only added there). + - **Anthropic** (and Bedrock Invoke via ``AnthropicConfig.transform_request``) + always get a dummy tool so CLIs work with ``modify_params`` left off. Reference Issue: https://github.com/BerriAI/litellm/issues/5747, https://github.com/BerriAI/litellm/issues/5388 """ - # Ensure modify_params is False so UnsupportedParamsError is raised + # Ensure modify_params is False so Bedrock Converse path still raises. # (other tests in this file set it to True and don't reset it) original_modify_params = litellm.modify_params litellm.modify_params = False try: litellm.set_verbose = True - messages = messages - - if expected_error_msg: + if expect_unsupported_params_error: with pytest.raises(litellm.UnsupportedParamsError) as e: second_response = litellm.completion( model=model, diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index e38698c9100..a19752dc648 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -2135,6 +2135,53 @@ def test_validate_effort_for_model_centralises_per_model_gating( assert err is None +def test_transform_request_injects_dummy_tool_without_tools_param(): + """ + Anthropic rejects messages that contain tool turns when ``tools`` is omitted. + LiteLLM must inject a dummy tool without ``litellm.modify_params``. + """ + config = AnthropicConfig() + prev_modify_params = litellm.modify_params + litellm.modify_params = False + try: + messages = [ + {"role": "user", "content": "Hello"}, + { + "role": "assistant", + "content": "Calling tool", + "tool_calls": [ + { + "id": "toolu_test_dummy", + "type": "function", + "function": {"name": "get_x", "arguments": "{}"}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "toolu_test_dummy", + "content": "{}", + }, + ] + result = config.transform_request( + model="claude-3-5-haiku-20241022", + messages=messages, + optional_params={"max_tokens": 256}, + litellm_params={}, + headers={}, + ) + finally: + litellm.modify_params = prev_modify_params + + assert "tools" in result + names = [ + t.get("name") + for t in result["tools"] + if isinstance(t, dict) and t.get("name") is not None + ] + assert "dummy_tool" in names + + def test_transform_request_uses_dynamic_max_tokens(): """ Test that transform_request uses dynamic max_tokens based on model From 5e016f9f74886ed78746c822f659ff6e57a2c046 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 11 May 2026 22:54:34 +0530 Subject: [PATCH 72/85] =?UTF-8?q?fix(responses):=20normalize=20chat=20tool?= =?UTF-8?q?=5Fchoice=20for=20completions=E2=86=92responses=20bridge=20(#27?= =?UTF-8?q?634)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(responses): map chat tool_choice to Responses API when bridging from completions OpenAI /v1/responses rejects tool_choice.function. Normalize forced-function choice from chat shape to {type, name} in LiteLLMResponsesTransformationHandler. Co-authored-by: Cursor * fix(responses): strip tool_choice.function when top-level name is set --------- Co-authored-by: Cursor --- .../transformation.py | 18 +++++++ ...responses_transformation_transformation.py | 50 +++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index da3b9184edb..32423f23314 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -119,6 +119,20 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): def __init__(self): pass + def _normalize_tool_choice_for_responses_api(self, tool_choice: Any) -> Any: + """Chat tool_choice uses function.name; Responses API expects top-level name.""" + if not isinstance(tool_choice, dict) or tool_choice.get("type") != "function": + return tool_choice + if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"): + # Return only Responses shape so stray chat ``function`` key is not sent upstream. + return {"type": "function", "name": tool_choice["name"]} + fn = tool_choice.get("function") + if isinstance(fn, dict): + fn_name = fn.get("name") + if isinstance(fn_name, str) and fn_name: + return {"type": "function", "name": fn_name} + return tool_choice + def _handle_raw_dict_response_item( self, item: Dict[str, Any], index: int ) -> Tuple[Optional[Any], int]: @@ -309,6 +323,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): text_format = self._transform_response_format_to_text_format(value) if text_format: responses_api_request["text"] = text_format # type: ignore + elif key == "tool_choice": + responses_api_request["tool_choice"] = ( # type: ignore[assignment] + self._normalize_tool_choice_for_responses_api(value) + ) elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys(): responses_api_request[key] = value # type: ignore elif key == "previous_response_id": diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index e40543e01a0..697a9ebc720 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -2098,6 +2098,56 @@ def test_map_optional_params_preserves_reasoning_summary(): assert responses_api_request["reasoning"]["summary"] == "detailed" +def test_map_optional_params_tool_choice_chat_nested_to_responses_api(): + """Chat tool_choice must become Responses ToolChoiceFunction (top-level name).""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams + + handler = LiteLLMResponsesTransformationHandler() + responses_api_request = ResponsesAPIOptionalRequestParams() + handler._map_optional_params_to_responses_api_request( + { + "stream": False, + "tool_choice": { + "type": "function", + "function": {"name": "Echo"}, + }, + }, + responses_api_request, + ) + assert responses_api_request["tool_choice"] == { + "type": "function", + "name": "Echo", + } + + +@pytest.mark.parametrize( + ("tool_choice", "expected"), + [ + ("auto", "auto"), + ("none", "none"), + ( + {"type": "function", "name": "Echo"}, + {"type": "function", "name": "Echo"}, + ), + ( + {"type": "function", "name": "foo", "function": {"name": "bar"}}, + {"type": "function", "name": "foo"}, + ), + ({"type": "required"}, {"type": "required"}), + ], +) +def test_normalize_tool_choice_for_responses_api(tool_choice, expected): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + assert handler._normalize_tool_choice_for_responses_api(tool_choice) == expected + + def test_convert_chat_completion_file_type_to_input_file(): """ Test that Chat Completion content with type 'file' is correctly mapped From 12e59c87984ca884d3fc874d32b88a9c2638e850 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Mon, 11 May 2026 10:44:50 -0700 Subject: [PATCH 73/85] Fix internal tag usage scoping (#27315) * Scope internal tag usage to own keys Co-authored-by: ishaan-berri * Add internal tag usage unowned key regression test Co-authored-by: ishaan-berri * Handle empty internal tag usage scopes safely Co-authored-by: ishaan-berri * Add tag activity database guard Co-authored-by: ishaan-berri --------- Co-authored-by: oss-agent-shin <279349115+oss-agent-shin@users.noreply.github.com> Co-authored-by: ishaan-berri --- litellm/proxy/_types.py | 15 +- .../tag_management_endpoints.py | 133 ++++++-- .../proxy/auth/test_route_checks.py | 35 +++ .../test_tag_management_endpoints.py | 291 ++++++++++++++++++ .../components/UsagePageView.test.tsx | 30 +- .../UsagePage/components/UsagePageView.tsx | 10 +- .../UsageViewSelect/UsageViewSelect.test.tsx | 12 + .../UsageViewSelect/UsageViewSelect.tsx | 5 + ui/litellm-dashboard/src/utils/roles.ts | 2 +- 9 files changed, 507 insertions(+), 26 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7f5a0da1066..6128f485748 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -677,6 +677,10 @@ class LiteLLMRoutes(enum.Enum): "/global/activity", "/global/activity/model", "/global/activity/cache_hits", + # Tag usage endpoints scope internal users to tags produced by + # their own keys in tag_management_endpoints.py. + "/tag/daily/activity", + "/tag/list", "/v1/models/{model_id}", "/models/{model_id}", "/guardrails/list", @@ -689,7 +693,16 @@ class LiteLLMRoutes(enum.Enum): + compliance_check_routes ) - internal_user_view_only_routes = spend_tracking_routes + compliance_check_routes + internal_user_view_only_routes = ( + spend_tracking_routes + + compliance_check_routes + + [ + # Tag usage endpoints scope internal viewers to tags produced by + # their own keys in tag_management_endpoints.py. + "/tag/daily/activity", + "/tag/list", + ] + ) self_managed_routes = [ "/team/member_add", diff --git a/litellm/proxy/management_endpoints/tag_management_endpoints.py b/litellm/proxy/management_endpoints/tag_management_endpoints.py index 2a4895d0299..49d9b67a28a 100644 --- a/litellm/proxy/management_endpoints/tag_management_endpoints.py +++ b/litellm/proxy/management_endpoints/tag_management_endpoints.py @@ -13,12 +13,12 @@ All /tag management endpoints import asyncio import json from datetime import datetime -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from fastapi import APIRouter, Depends, HTTPException, Query from litellm._logging import verbose_proxy_logger -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import UserAPIKeyAuth, user_api_key_has_admin_view from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.management_endpoints.common_daily_activity import ( SpendAnalyticsPaginatedResponse, @@ -40,6 +40,72 @@ if TYPE_CHECKING: router = APIRouter() +async def _get_internal_user_api_keys( + prisma_client, + user_api_key_dict: UserAPIKeyAuth, +) -> List[str]: + user_role = user_api_key_dict.user_role + if user_role is None or not user_role.is_internal_user_role: + return [] + + user_api_keys = set() + if user_api_key_dict.api_key: + user_api_keys.add(user_api_key_dict.api_key) + + user_id = user_api_key_dict.user_id + if user_id is None: + return sorted(user_api_keys) + + key_records = await prisma_client.db.litellm_verificationtoken.find_many( + where={"user_id": user_id}, + select={"token": True}, + ) + user_api_keys.update( + key_record.token + for key_record in key_records + if getattr(key_record, "token", None) + ) + + return sorted(user_api_keys) + + +async def _get_tag_list_scope( + prisma_client, + user_api_key_dict: UserAPIKeyAuth, +) -> Optional[Dict[str, dict]]: + user_role = user_api_key_dict.user_role + if user_api_key_has_admin_view(user_api_key_dict) or ( + user_role is None or not user_role.is_internal_user_role + ): + return None + + scoped_api_keys = await _get_internal_user_api_keys( + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + ) + return {"api_key": {"in": scoped_api_keys}} + + +async def _get_tag_daily_activity_api_key_filter( + prisma_client, + user_api_key_dict: UserAPIKeyAuth, + requested_api_key: Optional[str], +) -> Optional[Union[str, List[str]]]: + user_role = user_api_key_dict.user_role + if user_api_key_has_admin_view(user_api_key_dict) or ( + user_role is None or not user_role.is_internal_user_role + ): + return requested_api_key + + scoped_api_keys = await _get_internal_user_api_keys( + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + ) + if requested_api_key is not None: + return requested_api_key if requested_api_key in scoped_api_keys else [] + return scoped_api_keys + + async def _get_model_names(prisma_client, model_ids: list) -> Dict[str, str]: """Helper function to get model names from model IDs""" try: @@ -453,9 +519,41 @@ async def list_tags( _validate_tag_list_date_range(start_date, end_date) try: + tag_scope = await _get_tag_list_scope( + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + ) + + ## QUERY DYNAMIC TAGS ## + # Use group_by instead of find_many(distinct=["tag"]). + # Prisma's distinct fetches all columns for all rows and deduplicates + # in application code, which is extremely slow on large tables. + # See: https://www.prisma.io/docs/orm/prisma-client/queries/aggregation-grouping-summarizing#distinct-under-the-hood + dynamic_tag_where: Dict[str, Any] = {"tag": {"not": None}} + if tag_scope: + dynamic_tag_where = {**dynamic_tag_where, **tag_scope} + if start_date is not None and end_date is not None: + dynamic_tag_where["date"] = {"gte": start_date, "lte": end_date} + + dynamic_tag_rows = await prisma_client.db.litellm_dailytagspend.group_by( + by=["tag"], + where=dynamic_tag_where, + min={"created_at": True}, + max={"updated_at": True}, + ) + + used_tag_names = [row["tag"] for row in dynamic_tag_rows if row["tag"]] + if tag_scope is not None and not used_tag_names: + return [] + + stored_tag_where = ( + {"tag_name": {"in": used_tag_names}} if tag_scope is not None else None + ) + ## QUERY STORED TAGS ## tag_records = await prisma_client.db.litellm_tagtable.find_many( - include={"litellm_budget_table": True} + where=stored_tag_where, + include={"litellm_budget_table": True}, ) stored_tag_names = set() @@ -489,22 +587,6 @@ async def list_tags( list_of_tags.append(tag_dict) - ## QUERY DYNAMIC TAGS ## - # Use group_by instead of find_many(distinct=["tag"]). - # Prisma's distinct fetches all columns for all rows and deduplicates - # in application code, which is extremely slow on large tables. - # See: https://www.prisma.io/docs/orm/prisma-client/queries/aggregation-grouping-summarizing#distinct-under-the-hood - dynamic_tag_where: Dict[str, Any] = {"tag": {"not": None}} - if start_date is not None and end_date is not None: - dynamic_tag_where["date"] = {"gte": start_date, "lte": end_date} - - dynamic_tag_rows = await prisma_client.db.litellm_dailytagspend.group_by( - by=["tag"], - where=dynamic_tag_where, - min={"created_at": True}, - max={"updated_at": True}, - ) - dynamic_tag_config = [ { "name": row["tag"], @@ -572,6 +654,7 @@ async def get_tag_daily_activity( api_key: Optional[str] = None, page: int = 1, page_size: int = 10, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ Get daily activity for specific tags or all tags. @@ -590,8 +673,18 @@ async def get_tag_daily_activity( """ from litellm.proxy.proxy_server import prisma_client + if prisma_client is None: + raise HTTPException(status_code=500, detail="Database not connected") + # Convert comma-separated tags string to list if provided tag_list = tags.split(",") if tags else None + scoped_api_key_filter = await _get_tag_daily_activity_api_key_filter( + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + requested_api_key=api_key, + ) + if scoped_api_key_filter == []: + return SpendAnalyticsPaginatedResponse(results=[]) return await get_daily_activity( prisma_client=prisma_client, @@ -602,7 +695,7 @@ async def get_tag_daily_activity( start_date=start_date, end_date=end_date, model=model, - api_key=api_key, + api_key=scoped_api_key_filter, page=page, page_size=page_size, # metadata_metrics_func=None because litellm_dailytagspend rows are diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 3e0b1b739ec..fa56383e9bb 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -1889,6 +1889,41 @@ def test_non_admin_non_team_admin_cannot_access_config_update_but_can_attempt_re assert "Only proxy admin can be used to generate" in str(exc_info.value) +@pytest.mark.parametrize( + "user_role", + [ + LitellmUserRoles.INTERNAL_USER.value, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, + ], +) +@pytest.mark.parametrize("route", ["/tag/list", "/tag/daily/activity"]) +def test_internal_users_can_access_scoped_tag_usage_routes(user_role, route): + """ + Internal users can read tag usage endpoints because the endpoint handlers + scope results to the caller's own keys. + """ + user_obj = LiteLLM_UserTable( + user_id="test_user", + user_email="test@example.com", + user_role=user_role, + ) + valid_token = UserAPIKeyAuth( + user_id="test_user", + user_role=user_role, + ) + request = MagicMock(spec=Request) + request.query_params = {} + + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=user_role, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + + @pytest.mark.parametrize( "user_role", [ diff --git a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py index ee2d72d2dd4..76ba0e3dc67 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py @@ -380,6 +380,94 @@ async def test_list_tags_no_dynamic_tags(): app.dependency_overrides.clear() +async def test_internal_user_list_tags_only_returns_tags_used_by_their_keys(): + """ + Internal users can view tag usage, but the tag list must be scoped to tags + produced by API keys owned by the caller. + """ + from datetime import datetime + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + mock_user_auth = UserAPIKeyAuth( + api_key="current-owned-key", + user_id="internal-user-123", + user_role=LitellmUserRoles.INTERNAL_USER, + ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + + try: + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + mock_db = Mock() + mock_prisma.db = mock_db + + owned_key_record = Mock() + owned_key_record.token = "owned-key" + mock_db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[owned_key_record] + ) + + mock_db.litellm_dailytagspend.group_by = AsyncMock( + return_value=[ + { + "tag": "stored-owned-tag", + "_min": {"created_at": "2025-02-01T00:00:00Z"}, + "_max": {"updated_at": "2025-03-01T00:00:00Z"}, + }, + { + "tag": "dynamic-owned-tag", + "_min": {"created_at": "2025-02-02T00:00:00Z"}, + "_max": {"updated_at": "2025-03-02T00:00:00Z"}, + }, + ] + ) + + stored_tag = Mock() + stored_tag.tag_name = "stored-owned-tag" + stored_tag.description = "A stored tag used by the caller" + stored_tag.models = ["model-1"] + stored_tag.model_info = {} + stored_tag.spend = 0.0 + stored_tag.budget_id = None + stored_tag.created_at = datetime(2025, 1, 1) + stored_tag.updated_at = datetime(2025, 1, 1) + stored_tag.created_by = "admin-user" + stored_tag.litellm_budget_table = None + mock_db.litellm_tagtable.find_many = AsyncMock(return_value=[stored_tag]) + + response = client.get( + "/tag/list", + headers={"Authorization": "Bearer test-key"}, + ) + + assert response.status_code == 200 + assert [tag["name"] for tag in response.json()] == [ + "stored-owned-tag", + "dynamic-owned-tag", + ] + mock_db.litellm_verificationtoken.find_many.assert_awaited_once_with( + where={"user_id": "internal-user-123"}, + select={"token": True}, + ) + mock_db.litellm_dailytagspend.group_by.assert_awaited_once_with( + by=["tag"], + where={ + "tag": {"not": None}, + "api_key": {"in": ["current-owned-key", "owned-key"]}, + }, + min={"created_at": True}, + max={"updated_at": True}, + ) + mock_db.litellm_tagtable.find_many.assert_awaited_once_with( + where={"tag_name": {"in": ["stored-owned-tag", "dynamic-owned-tag"]}}, + include={"litellm_budget_table": True}, + ) + + finally: + app.dependency_overrides.clear() + + @pytest.mark.asyncio async def test_list_tags_with_date_range_filters_dynamic_tags(): """ @@ -420,6 +508,209 @@ async def test_list_tags_with_date_range_filters_dynamic_tags(): app.dependency_overrides.clear() +@pytest.mark.asyncio +async def test_internal_user_tag_daily_activity_is_scoped_to_their_keys(): + """ + Internal users must not receive proxy-wide tag spend rows when viewing tag + usage daily activity. + """ + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.management_endpoints.tag_management_endpoints import ( + get_tag_daily_activity, + ) + + mock_user_auth = UserAPIKeyAuth( + user_id="internal-user-123", + user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.management_endpoints.tag_management_endpoints.get_daily_activity", + new_callable=AsyncMock, + ) as mock_get_daily_activity, + ): + mock_db = Mock() + mock_prisma.db = mock_db + + owned_key_record = Mock() + owned_key_record.token = "owned-key" + mock_db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[owned_key_record] + ) + mock_get_daily_activity.return_value = "daily-activity-response" + + result = await get_tag_daily_activity( + start_date="2025-01-01", + end_date="2025-01-31", + user_api_key_dict=mock_user_auth, + ) + + assert result == "daily-activity-response" + mock_get_daily_activity.assert_awaited_once() + assert mock_get_daily_activity.await_args.kwargs["api_key"] == ["owned-key"] + + +@pytest.mark.asyncio +async def test_internal_user_tag_daily_activity_rejects_unowned_api_key_filter(): + """ + If an internal user filters tag usage by an API key they do not own, the + endpoint should return an empty scoped filter instead of exposing that key's + tag spend. + """ + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.management_endpoints.tag_management_endpoints import ( + get_tag_daily_activity, + ) + + mock_user_auth = UserAPIKeyAuth( + user_id="internal-user-123", + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.management_endpoints.tag_management_endpoints.get_daily_activity", + new_callable=AsyncMock, + ) as mock_get_daily_activity, + ): + mock_db = Mock() + mock_prisma.db = mock_db + + owned_key_record = Mock() + owned_key_record.token = "owned-key" + mock_db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[owned_key_record] + ) + result = await get_tag_daily_activity( + start_date="2025-01-01", + end_date="2025-01-31", + api_key="unowned-key", + user_api_key_dict=mock_user_auth, + ) + + assert result.results == [] + assert result.metadata.total_spend == 0 + assert result.metadata.total_api_requests == 0 + mock_get_daily_activity.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_internal_user_tag_daily_activity_scopes_to_current_key_without_user_id(): + """ + If an internal-user token has no user_id, it should still scope tag usage to + the current request key instead of falling back to proxy-wide tag spend. + """ + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.management_endpoints.tag_management_endpoints import ( + get_tag_daily_activity, + ) + + mock_user_auth = UserAPIKeyAuth( + api_key="current-owned-key", + user_id=None, + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.management_endpoints.tag_management_endpoints.get_daily_activity", + new_callable=AsyncMock, + ) as mock_get_daily_activity, + ): + mock_db = Mock() + mock_prisma.db = mock_db + mock_db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_get_daily_activity.return_value = "daily-activity-response" + + result = await get_tag_daily_activity( + start_date="2025-01-01", + end_date="2025-01-31", + user_api_key_dict=mock_user_auth, + ) + + assert result == "daily-activity-response" + mock_db.litellm_verificationtoken.find_many.assert_not_awaited() + mock_get_daily_activity.assert_awaited_once() + assert mock_get_daily_activity.await_args.kwargs["api_key"] == [ + "current-owned-key" + ] + + +@pytest.mark.asyncio +async def test_internal_user_tag_daily_activity_without_any_scoped_keys_returns_empty(): + """ + If an internal-user token has neither user_id nor api_key, the endpoint must + return an empty response instead of dropping the API key filter. + """ + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.management_endpoints.tag_management_endpoints import ( + get_tag_daily_activity, + ) + + mock_user_auth = UserAPIKeyAuth( + user_id=None, + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.management_endpoints.tag_management_endpoints.get_daily_activity", + new_callable=AsyncMock, + ) as mock_get_daily_activity, + ): + mock_db = Mock() + mock_prisma.db = mock_db + mock_db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + + result = await get_tag_daily_activity( + start_date="2025-01-01", + end_date="2025-01-31", + user_api_key_dict=mock_user_auth, + ) + + assert result.results == [] + assert result.metadata.total_spend == 0 + assert result.metadata.total_api_requests == 0 + mock_db.litellm_verificationtoken.find_many.assert_not_awaited() + mock_get_daily_activity.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_get_tag_daily_activity_requires_database_connection(): + """ + Tag daily activity should fail with the same explicit DB error used by other + tag endpoints instead of raising an AttributeError during scope resolution. + """ + from litellm.proxy.management_endpoints.tag_management_endpoints import ( + get_tag_daily_activity, + ) + + mock_user_auth = UserAPIKeyAuth( + user_id="internal-user-123", + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + with patch("litellm.proxy.proxy_server.prisma_client", None): + with pytest.raises(HTTPException) as exc_info: + await get_tag_daily_activity( + start_date="2025-01-01", + end_date="2025-01-31", + user_api_key_dict=mock_user_auth, + ) + + assert exc_info.value.status_code == 500 + assert exc_info.value.detail == "Database not connected" + + @pytest.mark.asyncio async def test_list_tags_without_date_range_omits_date_filter(): """When no date range is passed, the WHERE clause must not carry a date key.""" diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.test.tsx index bbcddd572cd..b95f63368c7 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.test.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.test.tsx @@ -57,7 +57,10 @@ vi.mock("./EndpointUsage/EndpointUsage", () => ({ vi.mock("./UsageViewSelect/UsageViewSelect", async () => { const React = await import("react"); - const UsageViewSelect = ({ value, onChange }: any) => { + const UsageViewSelect = ({ value, onChange, canViewTagUsage = false }: any) => { + const tagOption = canViewTagUsage + ? React.createElement("option", { value: "tag" }, "Tag Usage") + : null; return React.createElement( "select", { @@ -70,7 +73,7 @@ vi.mock("./UsageViewSelect/UsageViewSelect", async () => { React.createElement("option", { value: "team" }, "Team Usage"), React.createElement("option", { value: "organization" }, "Organization Usage"), React.createElement("option", { value: "customer" }, "Customer Usage"), - React.createElement("option", { value: "tag" }, "Tag Usage"), + tagOption, React.createElement("option", { value: "agent" }, "Agent Usage"), React.createElement("option", { value: "user-agent-activity" }, "User Agent Activity"), ); @@ -639,6 +642,29 @@ describe("UsagePage", () => { }); }); + it("should show tag usage selector option for internal users", async () => { + mockUseAuthorized.mockReturnValue({ + isLoading: false, + isAuthorized: true, + token: "mock-token", + accessToken: "test-token", + userId: "user-123", + userEmail: "test@example.com", + userRole: "internal_user", + premiumUser: true, + disabledPersonalKeyCreation: false, + showSSOBanner: false, + }); + + renderWithProviders(); + + await waitFor(() => { + expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalled(); + }); + + expect(screen.getByRole("option", { name: "Tag Usage" })).toBeInTheDocument(); + }); + it("should show organization usage banner and view for admins", async () => { renderWithProviders(); diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx index efc1166a398..50dd5e78f37 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx @@ -31,7 +31,7 @@ import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { useCurrentUser } from "@/app/(dashboard)/hooks/users/useCurrentUser"; import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { all_admin_roles } from "../../../utils/roles"; +import { all_admin_roles, internalUserRoles } from "../../../utils/roles"; import { ActivityMetrics, processActivityData } from "../../activity_metrics"; import CloudZeroExportModal from "../../cloudzero_export_modal"; import EntityUsageExportModal from "../../EntityUsageExport"; @@ -84,6 +84,7 @@ const UsagePage: React.FC = ({ teams, organizations }) => { console.log(`currentUser: ${JSON.stringify(currentUser)}`); console.log(`currentUser max budget: ${currentUser?.max_budget}`); const isAdmin = all_admin_roles.includes(userRole || ""); + const canViewTagUsage = isAdmin || internalUserRoles.includes(userRole || ""); // Debounced search for user selector const [userSearchInput, setUserSearchInput] = useState(""); @@ -444,7 +445,12 @@ const UsagePage: React.FC = ({ teams, organizations }) => {

- setUsageView(value)} isAdmin={isAdmin} /> + setUsageView(value)} + isAdmin={isAdmin} + canViewTagUsage={canViewTagUsage} + />
{paginatedResult.isFetchingMore && ( diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.test.tsx index 7bb80b424b5..b33129fcdb4 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.test.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.test.tsx @@ -110,4 +110,16 @@ describe("UsageViewSelect", () => { expect(mockOnChange).toHaveBeenCalledWith("team"); }); + + it("should show Tag Usage for non-admin users with tag usage permission", () => { + render(); + + expect(screen.getByRole("option", { name: "Tag Usage" })).toBeInTheDocument(); + }); + + it("should hide Tag Usage for non-admin users without tag usage permission", () => { + render(); + + expect(screen.queryByRole("option", { name: "Tag Usage" })).not.toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.tsx index 11fdb8a7cf5..184fc000274 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.tsx @@ -16,6 +16,7 @@ export interface UsageViewSelectProps { value: UsageOption; onChange: (value: UsageOption) => void; isAdmin: boolean; + canViewTagUsage?: boolean; title?: string; description?: string; "data-id"?: string; @@ -106,12 +107,16 @@ export const UsageViewSelect: React.FC = ({ value, onChange, isAdmin, + canViewTagUsage = false, title = "Usage View", description = "Select the usage data you want to view", "data-id": dataId, }) => { const getFilteredOptions = () => { return OPTIONS.filter((option) => { + if (option.value === "tag" && canViewTagUsage) { + return true; + } if (option.adminOnly && !isAdmin) { return false; } diff --git a/ui/litellm-dashboard/src/utils/roles.ts b/ui/litellm-dashboard/src/utils/roles.ts index 11e31fd369c..bb23985d7c2 100644 --- a/ui/litellm-dashboard/src/utils/roles.ts +++ b/ui/litellm-dashboard/src/utils/roles.ts @@ -5,7 +5,7 @@ export const old_admin_roles = ["Admin", "Admin Viewer"]; export const v2_admin_role_names = ["proxy_admin", "proxy_admin_viewer", "org_admin"]; export const all_admin_roles = [...old_admin_roles, ...v2_admin_role_names]; -export const internalUserRoles = ["Internal User", "Internal Viewer"]; +export const internalUserRoles = ["Internal User", "Internal Viewer", "internal_user", "internal_user_viewer"]; export const rolesAllowedToSeeUsage = ["Admin", "Admin Viewer", "Internal User", "Internal Viewer"]; export const rolesWithWriteAccess = ["Internal User", "Admin", "proxy_admin"]; // Admin-tier read parity: Admin Viewer sees Models + Endpoints, Agents, and From 075188668055b813f4e1353371eaeaceb0395cc3 Mon Sep 17 00:00:00 2001 From: Dawei Gu <99218665+dgu1-godaddy@users.noreply.github.com> Date: Mon, 11 May 2026 13:22:26 -0700 Subject: [PATCH 74/85] feat(batch-job): bedrock batch model invocation job retrieval (#26834) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(bedrock): support retrieve for model-invocation-job batch ARNs `bedrock.retrieve_batch` previously only handled `:async-invoke/` ARNs (Twelve Labs Marengo embeddings). The `:model-invocation-job/` ARNs returned by `CreateModelInvocationJob` (the bulk batch inference API behind `bedrock.create_batch`) fell through and returned a misleading data-plane error, leaving created jobs unretrievable through the LiteLLM batches API. The two ARN families live on different AWS service endpoints (`bedrock-runtime` data plane vs `bedrock` control plane), so they need distinct handlers. This adds: * `BedrockBatchesHandler._handle_model_invocation_job_status` — calls the control plane via boto3 (`bedrock:GetModelInvocationJob`), reusing `BaseAWSLLM.get_credentials` for credential resolution so model_list / env / role-assumption configs continue to apply. The response is reshaped into a `LiteLLMBatch` with the same status mapping `transform_create_batch_response` already uses. * Output-file-URI prediction. Bedrock surfaces the user-supplied `s3OutputDataConfig.s3Uri` *prefix* in `GetModelInvocationJob`, but results actually land at `//.out`. We compute that single-file URI client-side and surface it as `output_file_id`, so OpenAI-style `client.files.content(...)` works without an extra `ListObjectsV2` round-trip. The bare prefix stays in metadata for callers that want the manifest. * Dispatch in `litellm/batches/main.py` for the new ARN family, alongside the existing async-invoke branch. * Unit tests covering ARN parsing, output-URI prediction (incl. edge cases), the full status mapping, region resolution precedence, and failure-message propagation. Note: `request_counts` is intentionally `(0, 0, 0)` — `GetModelInvocationJob` does not report per-record counts; getting accurate numbers requires parsing `manifest.json.out` from the output S3 prefix, which is left to callers. Made-with: Cursor * fix(bedrock): address PR feedback on model-invocation-job retrieve Addresses Greptile P2 findings on #26834: 1. Use the bare job id (not the full ARN) when constructing the `api_base` URL for `pre_call` logging. Passing the full ARN double- counts the `model-invocation-job/` segment and embeds colons in the path, producing misleading log lines. 2. Drop the `or output_prefix` fallback when `_predict_output_file_uri` returns None. A bare prefix is not a downloadable object and surfacing it as `output_file_id` re-creates the very NoSuchKey bug this handler exists to fix. The bare prefix is still preserved in `metadata["output_s3_uri"]` for callers that want to do their own S3 listing or read `manifest.json.out`. `metadata["output_file_uri"]` uses "" rather than None to satisfy the OpenAI Batch metadata schema (`dict[str, str]`); callers should branch on the typed `output_file_id` field instead. Also expands test coverage on the new code path: - new "stay None" regression test for the prediction-fail case - pre_call/post_call logging hook assertions (incl. the bare-id URL) - explicit cancelled_at / expired_at coverage - _to_epoch type-handling matrix and the boto3 ImportError branch - defensive _extract_region_from_bedrock_arn exception path - empty-basename case for _predict_output_file_uri Patch coverage on the changed lines is now 100% (the only remaining uncovered lines in the file belong to the pre-existing `_handle_async_invoke_status` method, which this PR does not touch). Made-with: Cursor * test(bedrock): cover retrieve_batch dispatch for both ARN families Codecov flagged 8 uncovered lines on `litellm/batches/main.py` after this PR refactored the Bedrock dispatch into a single guard with two sub-branches (`async-invoke` + `model-invocation-job`). Existing tests exercised the handlers directly but not the dispatch in `main.py`. Adds `tests/test_litellm/batches/test_retrieve_batch_bedrock_dispatch.py` with 6 mocked tests that exercise `litellm.retrieve_batch` end-to-end for the dispatch logic: - async-invoke ARN routes to `_handle_async_invoke_status` - async-invoke ARN with no region falls back to "us-east-1" (preserves prior behavior on this branch) - model-invocation-job ARN routes to the new `_handle_model_invocation_job_status` handler - model-invocation-job ARN with no region forwards None (so the new handler can sniff region from the ARN itself, rather than getting silently routed to us-east-1) - unrelated bedrock ARN family falls through to the generic provider-config retrieve path (neither special handler invoked) - non-bedrock batch ids skip the bedrock dispatch entirely Both handlers are mocked at the import site so the tests don't hit AWS — the focus here is purely the new dispatch logic in main.py. Co-authored-by: Cursor * test(bedrock): move retrieve_batch dispatch test to tests/test_litellm/ The dispatch test landed under `tests/test_litellm/batches/`, a new directory that no upstream `test-unit-*.yml` workflow's `test-path` allow-list includes. As a result, the test was never executed in CI and codecov reported `litellm/batches/main.py` patch coverage at 11.11% (8 lines uncovered) — the lines belonging to this PR's dispatch refactor itself. Move the file up one level so it matches the `tests/test_litellm/test_*.py` glob that `test-unit-misc.yml` already runs, and adjust `sys.path.insert` for the new depth. The companion handler tests under `tests/test_litellm/llms/bedrock/batches/test_handler.py` are unaffected — they're picked up by the `llms` directory in `test-unit-llm-providers.yml`. Made-with: Cursor --------- Co-authored-by: Cursor --- litellm/batches/main.py | 45 ++- litellm/llms/bedrock/batches/handler.py | 241 +++++++++++++ .../llms/bedrock/batches/test_handler.py | 338 ++++++++++++++++++ .../test_retrieve_batch_bedrock_dispatch.py | 162 +++++++++ 4 files changed, 769 insertions(+), 17 deletions(-) create mode 100644 tests/test_litellm/llms/bedrock/batches/test_handler.py create mode 100644 tests/test_litellm/test_retrieve_batch_bedrock_dispatch.py diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 259439d4d09..15ee9303969 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -617,24 +617,35 @@ def retrieve_batch( _is_async = kwargs.pop("aretrieve_batch", False) is True client = kwargs.get("client", None) - # Check if this is an async invoke ARN (different from regular batch ARN) - # Async invoke ARNs have format: arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}:[0-9]{12}:async-invoke/[a-z0-9]{12} - if ( - batch_id.startswith("arn:aws") - and ":bedrock:" in batch_id - and ":async-invoke/" in batch_id - ): - # Handle async invoke status check - # Remove aws_region_name from kwargs to avoid duplicate parameter - async_kwargs = kwargs.copy() - async_kwargs.pop("aws_region_name", None) + # Bedrock has two distinct ARN families that need different APIs: + # * async-invoke ARNs (Twelve Labs Marengo embeddings) -> bedrock-runtime data plane + # * model-invocation-job ARNs (CreateModelInvocationJob batch) -> bedrock control plane + # They live on different AWS service endpoints and can't share a handler. + # ARN shapes: + # arn:aws(-[^:]+)?:bedrock:::async-invoke/ + # arn:aws(-[^:]+)?:bedrock:::model-invocation-job/ + if batch_id.startswith("arn:aws") and ":bedrock:" in batch_id: + if ":async-invoke/" in batch_id: + # Remove aws_region_name from kwargs to avoid duplicate parameter + async_kwargs = kwargs.copy() + async_kwargs.pop("aws_region_name", None) - return BedrockBatchesHandler._handle_async_invoke_status( - batch_id=batch_id, - aws_region_name=kwargs.get("aws_region_name", "us-east-1"), - logging_obj=litellm_logging_obj, - **async_kwargs, - ) + return BedrockBatchesHandler._handle_async_invoke_status( + batch_id=batch_id, + aws_region_name=kwargs.get("aws_region_name", "us-east-1"), + logging_obj=litellm_logging_obj, + **async_kwargs, + ) + if ":model-invocation-job/" in batch_id: + mij_kwargs = kwargs.copy() + mij_kwargs.pop("aws_region_name", None) + + return BedrockBatchesHandler._handle_model_invocation_job_status( + batch_id=batch_id, + aws_region_name=kwargs.get("aws_region_name"), + logging_obj=litellm_logging_obj, + **mij_kwargs, + ) # Try to use provider config first (for providers like bedrock) model: Optional[str] = kwargs.get("model", None) diff --git a/litellm/llms/bedrock/batches/handler.py b/litellm/llms/bedrock/batches/handler.py index f141bbd9ab4..c071f331337 100644 --- a/litellm/llms/bedrock/batches/handler.py +++ b/litellm/llms/bedrock/batches/handler.py @@ -1,8 +1,79 @@ +from datetime import datetime +from typing import Any, Optional, cast + from openai.types.batch import BatchRequestCounts from openai.types.batch import Metadata as OpenAIBatchMetadata from litellm.types.utils import LiteLLMBatch +# AWS Bedrock model-invocation-job statuses → OpenAI Batch statuses. +# Mirrors the mapping used by `BedrockBatchesConfig.transform_create_batch_response` +# so create / retrieve return consistent statuses. +_BEDROCK_MIJ_STATUS_TO_OPENAI = { + "Submitted": "validating", + "Validating": "validating", + "Scheduled": "validating", + "InProgress": "in_progress", + "Stopping": "cancelling", + "Stopped": "cancelled", + "Completed": "completed", + "PartiallyCompleted": "completed", + "Failed": "failed", + "Expired": "expired", +} + + +def _extract_region_from_bedrock_arn(arn: str) -> Optional[str]: + """ARN shape: ``arn:aws:bedrock:::/``""" + try: + parts = arn.split(":") + if len(parts) >= 4 and parts[2] == "bedrock": + return parts[3] or None + except Exception: + pass + return None + + +def _extract_job_id_from_arn(arn: str) -> Optional[str]: + """``arn:aws:bedrock:::model-invocation-job/`` -> ````.""" + if ":model-invocation-job/" not in arn: + return None + return arn.rsplit("/", 1)[-1] or None + + +def _predict_output_file_uri( + output_prefix: str, input_uri: str, job_id: Optional[str] +) -> Optional[str]: + """ + Compute the deterministic per-job result file URI Bedrock writes to. + + Bedrock lays results out as:: + + //.out + + We compute it client-side so OpenAI-style ``client.files.content(output_file_id)`` + works without an extra S3 ``ListObjectsV2`` round-trip. Returns ``None`` if we + don't have enough info; callers should fall back to the bare prefix. + """ + if not output_prefix or not input_uri or not job_id: + return None + if not output_prefix.endswith("/"): + output_prefix = output_prefix + "/" + input_basename = input_uri.rsplit("/", 1)[-1] + if not input_basename: + return None + return f"{output_prefix}{job_id}/{input_basename}.out" + + +def _to_epoch(value: Any) -> Optional[int]: + if value is None: + return None + if isinstance(value, (int, float)): + return int(value) + if isinstance(value, datetime): + return int(value.timestamp()) + return None + class BedrockBatchesHandler: """ @@ -97,3 +168,173 @@ class BedrockBatchesHandler: with concurrent.futures.ThreadPoolExecutor() as executor: future = executor.submit(run_in_thread) return future.result() + + @staticmethod + def _handle_model_invocation_job_status( + batch_id: str, + aws_region_name: Optional[str] = None, + logging_obj=None, + **kwargs, + ) -> "LiteLLMBatch": + """ + Handle ``GetModelInvocationJob`` status check for AWS Bedrock bulk batch + inference jobs (the ARN type returned by ``CreateModelInvocationJob``). + + ``CreateModelInvocationJob`` lives on the Bedrock **control plane** + (``bedrock..amazonaws.com``), distinct from the data-plane + ``bedrock-runtime`` endpoint that serves Twelve Labs async-invoke ARNs. + The two ARN families therefore can't share a handler — see + ``litellm/batches/main.py`` for the dispatch. + + Args: + batch_id: A ``arn:aws:bedrock:::model-invocation-job/`` + ARN (or just the trailing job id; both are accepted by + ``GetModelInvocationJob``). + aws_region_name: Region for the boto3 ``bedrock`` client. If omitted, + we fall back to parsing the region out of ``batch_id`` itself. + logging_obj: Optional litellm logging object. + **kwargs: Optional AWS credential overrides + (``aws_access_key_id``, ``aws_secret_access_key``, + ``aws_session_token``, ``aws_profile_name``, + ``aws_role_name``, ``aws_session_name``, + ``aws_web_identity_token``, ``aws_sts_endpoint``, + ``aws_external_id``). Unknown keys are ignored. + + Returns: + ``LiteLLMBatch`` shaped like an OpenAI Batch resource. Note that + ``request_counts`` is always ``(0, 0, 0)`` because + ``GetModelInvocationJob`` does not surface per-record counts; + callers that need accurate counts should parse + ``manifest.json.out`` from the output S3 prefix. + """ + try: + import boto3 + except ImportError as exc: + raise ImportError( + "Missing boto3 to call bedrock. Run 'pip install boto3'." + ) from exc + + # Resolve region: explicit > parsed-from-ARN > us-east-1 (boto3 default). + region = ( + aws_region_name or _extract_region_from_bedrock_arn(batch_id) or "us-east-1" + ) + + # Resolve credentials through the same path the rest of the bedrock + # provider uses, so model_list / env / role-assumption configs are + # honored. We instantiate BedrockBatchesConfig (which extends + # BaseAWSLLM) lazily to avoid a circular import at module load. + from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig + + creds = BedrockBatchesConfig().get_credentials( + aws_access_key_id=kwargs.get("aws_access_key_id"), + aws_secret_access_key=kwargs.get("aws_secret_access_key"), + aws_session_token=kwargs.get("aws_session_token"), + aws_region_name=region, + aws_session_name=kwargs.get("aws_session_name"), + aws_profile_name=kwargs.get("aws_profile_name"), + aws_role_name=kwargs.get("aws_role_name"), + aws_web_identity_token=kwargs.get("aws_web_identity_token"), + aws_sts_endpoint=kwargs.get("aws_sts_endpoint"), + aws_external_id=kwargs.get("aws_external_id"), + ) + + client = boto3.client( + "bedrock", + region_name=region, + aws_access_key_id=creds.access_key, + aws_secret_access_key=creds.secret_key, + aws_session_token=creds.token, + ) + + if logging_obj is not None: + # Use the bare job id in the logged URL so we don't double up the + # `model-invocation-job/` segment when `batch_id` is a full ARN. + # `GetModelInvocationJob` accepts either form, but only the bare id + # produces a sensible-looking URL in logs. + url_path_id = _extract_job_id_from_arn(batch_id) or batch_id + logging_obj.pre_call( + input=batch_id, + api_key="", + additional_args={ + "complete_input_dict": {"jobIdentifier": batch_id}, + "api_base": ( + f"https://bedrock.{region}.amazonaws.com/" + f"model-invocation-job/{url_path_id}" + ), + }, + ) + + response = client.get_model_invocation_job(jobIdentifier=batch_id) + + if logging_obj is not None: + logging_obj.post_call( + input=batch_id, + api_key="", + original_response=response, + additional_args={"complete_input_dict": {"jobIdentifier": batch_id}}, + ) + + bedrock_status = str(response.get("status", "")) + openai_status = cast( + Any, + _BEDROCK_MIJ_STATUS_TO_OPENAI.get(bedrock_status, "in_progress"), + ) + + input_uri = ( + response.get("inputDataConfig", {}) + .get("s3InputDataConfig", {}) + .get("s3Uri", "") + ) + output_prefix = ( + response.get("outputDataConfig", {}) + .get("s3OutputDataConfig", {}) + .get("s3Uri", "") + ) + + # Bedrock returns the output *prefix* the user supplied at job creation. + # Actual results land at //.out — we + # surface that single-file URI as `output_file_id` so the OpenAI-style + # download flow works without an extra S3 listing call. We deliberately + # do NOT fall back to the bare prefix when prediction fails: a prefix + # is not a downloadable object, so handing it back as `output_file_id` + # would reproduce the very NoSuchKey bug this handler exists to fix. + # The bare prefix is preserved in metadata for callers that want the + # `manifest.json.out` or want to do their own listing. + job_arn = response.get("jobArn", batch_id) + job_id = _extract_job_id_from_arn(job_arn) + output_file_uri = _predict_output_file_uri(output_prefix, input_uri, job_id) + + completed_at = _to_epoch(response.get("endTime")) + + # Note: metadata uses "" (not None) for unknown URIs to satisfy the + # OpenAI Batch metadata schema, which is `dict[str, str]`. The + # `output_file_id` field on the LiteLLMBatch itself does carry None + # correctly (see below), so callers should branch on that, not on + # `metadata["output_file_uri"]`. + openai_batch_metadata: OpenAIBatchMetadata = { + "model_arn": response.get("modelId", ""), + "job_arn": job_arn, + "job_name": response.get("jobName", ""), + "failure_message": response.get("message") or "", + "input_s3_uri": input_uri, + "output_s3_uri": output_prefix, + "output_file_uri": output_file_uri or "", + } + + return LiteLLMBatch( + id=job_arn, + object="batch", + status=openai_status, + created_at=_to_epoch(response.get("submitTime")) or 0, + in_progress_at=_to_epoch(response.get("lastModifiedTime")), + completed_at=completed_at if openai_status == "completed" else None, + failed_at=completed_at if openai_status == "failed" else None, + cancelled_at=completed_at if openai_status == "cancelled" else None, + expired_at=completed_at if openai_status == "expired" else None, + request_counts=BatchRequestCounts(total=0, completed=0, failed=0), + metadata=openai_batch_metadata, + completion_window="24h", + endpoint="/v1/chat/completions", + input_file_id=input_uri, + output_file_id=output_file_uri if openai_status == "completed" else None, + ) diff --git a/tests/test_litellm/llms/bedrock/batches/test_handler.py b/tests/test_litellm/llms/bedrock/batches/test_handler.py new file mode 100644 index 00000000000..18780ccce0f --- /dev/null +++ b/tests/test_litellm/llms/bedrock/batches/test_handler.py @@ -0,0 +1,338 @@ +"""Unit tests for ``BedrockBatchesHandler._handle_model_invocation_job_status``. + +These cover the upstream support for retrieving Bedrock bulk batch jobs +(``arn:aws:bedrock:::model-invocation-job/``) — the ARN +type returned by ``CreateModelInvocationJob``. We mock the boto3 client so +the tests don't hit AWS. +""" + +from __future__ import annotations + +import os +import sys +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.bedrock.batches.handler import ( # noqa: E402 + BedrockBatchesHandler, + _extract_job_id_from_arn, + _extract_region_from_bedrock_arn, + _predict_output_file_uri, + _to_epoch, +) + +JOB_ID = "abc1234567" +JOB_ARN = f"arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/{JOB_ID}" +INPUT_URI = "s3://my-bucket/inputs/qwen3-235b-a22b-2507-batch.jsonl" +OUTPUT_PREFIX = "s3://my-bucket/litellm-batch-outputs/litellm-bedrock-files-qwen-uuid/" +SUBMIT_TIME = datetime(2026, 4, 28, 12, 0, 0, tzinfo=timezone.utc) +END_TIME = datetime(2026, 4, 28, 12, 30, 0, tzinfo=timezone.utc) + + +def _fake_boto3_response(status: str = "Completed", end_time=END_TIME): + return { + "jobArn": JOB_ARN, + "jobName": "litellm-bedrock-files-qwen-uuid", + "modelId": "bedrock/qwen.qwen3-235b-a22b-2507-v1:0", + "status": status, + "submitTime": SUBMIT_TIME, + "lastModifiedTime": end_time, + "endTime": end_time, + "inputDataConfig": {"s3InputDataConfig": {"s3Uri": INPUT_URI}}, + "outputDataConfig": {"s3OutputDataConfig": {"s3Uri": OUTPUT_PREFIX}}, + } + + +@pytest.fixture +def patched_boto3(): + """Yield a stub bedrock client whose `get_model_invocation_job` is a MagicMock.""" + fake_client = MagicMock() + fake_client.get_model_invocation_job.return_value = _fake_boto3_response() + with ( + patch("boto3.client", return_value=fake_client) as boto_client_factory, + patch( + "litellm.llms.bedrock.batches.transformation.BedrockBatchesConfig.get_credentials", + return_value=MagicMock(access_key="AKIA", secret_key="SECRET", token=None), + ), + ): + yield fake_client, boto_client_factory + + +def test_extract_region_from_arn(): + assert _extract_region_from_bedrock_arn(JOB_ARN) == "us-west-2" + assert _extract_region_from_bedrock_arn("arn:aws:bedrock::123:foo/bar") is None + assert _extract_region_from_bedrock_arn("not-an-arn") is None + + +def test_extract_region_swallows_unexpected_split_errors(): + """Defensive `except Exception` branch — anything that isn't a plain str + should fall through to ``None`` rather than blow up.""" + + class WeirdArn: + def split(self, _sep): + raise RuntimeError("boom") + + assert _extract_region_from_bedrock_arn(WeirdArn()) is None # type: ignore[arg-type] + + +def test_predict_output_file_uri_returns_none_for_directory_input_uri(): + """Input URI ending in `/` has an empty basename — we must bail rather + than emit ``//.out``.""" + assert ( + _predict_output_file_uri(OUTPUT_PREFIX, "s3://bucket/inputs/", JOB_ID) is None + ) + + +_DT = datetime(2026, 4, 28, 12, 0, 0, tzinfo=timezone.utc) + + +@pytest.mark.parametrize( + "value,expected", + [ + (None, None), + (1730000000, 1730000000), + (1730000000.5, 1730000000), + (_DT, int(_DT.timestamp())), + ("2026-04-28T12:00:00Z", None), # strings aren't supported -> None + ], +) +def test_to_epoch_handles_supported_types(value, expected): + assert _to_epoch(value) == expected + + +def test_extract_job_id_from_arn(): + assert _extract_job_id_from_arn(JOB_ARN) == JOB_ID + assert ( + _extract_job_id_from_arn("arn:aws:bedrock:us-west-2:1:async-invoke/x") is None + ) + + +def test_predict_output_file_uri_happy_path(): + expected = f"{OUTPUT_PREFIX}{JOB_ID}/qwen3-235b-a22b-2507-batch.jsonl.out" + assert _predict_output_file_uri(OUTPUT_PREFIX, INPUT_URI, JOB_ID) == expected + + +def test_predict_output_file_uri_adds_trailing_slash(): + prefix_no_slash = OUTPUT_PREFIX.rstrip("/") + expected = f"{OUTPUT_PREFIX}{JOB_ID}/qwen3-235b-a22b-2507-batch.jsonl.out" + assert _predict_output_file_uri(prefix_no_slash, INPUT_URI, JOB_ID) == expected + + +@pytest.mark.parametrize( + "missing_arg", + [ + ("", INPUT_URI, JOB_ID), + (OUTPUT_PREFIX, "", JOB_ID), + (OUTPUT_PREFIX, INPUT_URI, None), + ], +) +def test_predict_output_file_uri_returns_none_when_missing_input(missing_arg): + assert _predict_output_file_uri(*missing_arg) is None + + +def test_handle_model_invocation_job_status_completed(patched_boto3): + fake_client, boto_client_factory = patched_boto3 + + batch = BedrockBatchesHandler._handle_model_invocation_job_status(batch_id=JOB_ARN) + + fake_client.get_model_invocation_job.assert_called_once_with(jobIdentifier=JOB_ARN) + + # Region should be sniffed from the ARN. + _, kwargs = boto_client_factory.call_args + assert kwargs["region_name"] == "us-west-2" + + assert batch.id == JOB_ARN + assert batch.status == "completed" + assert batch.input_file_id == INPUT_URI + expected_out = f"{OUTPUT_PREFIX}{JOB_ID}/qwen3-235b-a22b-2507-batch.jsonl.out" + assert batch.output_file_id == expected_out + assert batch.completed_at == int(END_TIME.timestamp()) + assert batch.failed_at is None + assert batch.cancelled_at is None + # Per-record counts aren't reported by GetModelInvocationJob, so we leave + # them zeroed; consumers should parse manifest.json.out for accurate counts. + assert batch.request_counts.total == 0 + assert batch.metadata["job_arn"] == JOB_ARN + assert batch.metadata["output_file_uri"] == expected_out + assert batch.metadata["output_s3_uri"] == OUTPUT_PREFIX + + +@pytest.mark.parametrize( + "bedrock_status,openai_status", + [ + ("Submitted", "validating"), + ("Validating", "validating"), + ("Scheduled", "validating"), + ("InProgress", "in_progress"), + ("Stopping", "cancelling"), + ("Stopped", "cancelled"), + ("Completed", "completed"), + ("PartiallyCompleted", "completed"), + ("Failed", "failed"), + ("Expired", "expired"), + # Unknown/unmapped Bedrock status falls back to "in_progress" so we + # don't 500 on a future AWS-side enum addition. + ("MyBrandNewStatus", "in_progress"), + ], +) +def test_status_mapping(patched_boto3, bedrock_status, openai_status): + fake_client, _ = patched_boto3 + fake_client.get_model_invocation_job.return_value = _fake_boto3_response( + status=bedrock_status + ) + + batch = BedrockBatchesHandler._handle_model_invocation_job_status(batch_id=JOB_ARN) + + assert batch.status == openai_status + # output_file_id is only populated for terminal-completed jobs, so callers + # don't accidentally try to download a non-existent file mid-run. + if openai_status == "completed": + assert batch.output_file_id is not None + else: + assert batch.output_file_id is None + + +def test_explicit_region_overrides_arn(patched_boto3): + _, boto_client_factory = patched_boto3 + BedrockBatchesHandler._handle_model_invocation_job_status( + batch_id=JOB_ARN, aws_region_name="eu-central-1" + ) + _, kwargs = boto_client_factory.call_args + assert kwargs["region_name"] == "eu-central-1" + + +def test_failure_message_propagates(patched_boto3): + fake_client, _ = patched_boto3 + failed_response = _fake_boto3_response(status="Failed") + failed_response["message"] = "Input file failed validation" + fake_client.get_model_invocation_job.return_value = failed_response + + batch = BedrockBatchesHandler._handle_model_invocation_job_status(batch_id=JOB_ARN) + + assert batch.status == "failed" + assert batch.failed_at == int(END_TIME.timestamp()) + assert batch.metadata["failure_message"] == "Input file failed validation" + + +def test_completed_with_unpredictable_output_uri_stays_none(patched_boto3): + """ + Regression guard for the original NoSuchKey bug: if Bedrock's response is + missing pieces we need to compute the per-job output file path (here, the + input s3Uri), `output_file_id` must stay `None` rather than fall back to + the bare prefix. Falling back to the prefix is what produced the original + NoSuchKey error this PR fixes. + """ + fake_client, _ = patched_boto3 + incomplete_response = _fake_boto3_response(status="Completed") + incomplete_response["inputDataConfig"] = {"s3InputDataConfig": {"s3Uri": ""}} + fake_client.get_model_invocation_job.return_value = incomplete_response + + batch = BedrockBatchesHandler._handle_model_invocation_job_status(batch_id=JOB_ARN) + + assert batch.status == "completed" + # output_file_id MUST be None (not the bare prefix) — that's the whole + # point of this regression test. Callers branch on this field. + assert batch.output_file_id is None + # The metadata field uses "" because OpenAI Batch metadata is dict[str, str]; + # callers should branch on `output_file_id` (above) instead. + assert batch.metadata["output_file_uri"] == "" + # The bare prefix is still preserved in metadata so callers can list it. + assert batch.metadata["output_s3_uri"] == OUTPUT_PREFIX + + +def test_cancelled_status_sets_cancelled_at(patched_boto3): + fake_client, _ = patched_boto3 + fake_client.get_model_invocation_job.return_value = _fake_boto3_response( + status="Stopped" + ) + + batch = BedrockBatchesHandler._handle_model_invocation_job_status(batch_id=JOB_ARN) + + assert batch.status == "cancelled" + assert batch.cancelled_at == int(END_TIME.timestamp()) + assert batch.completed_at is None + assert batch.failed_at is None + assert batch.expired_at is None + + +def test_expired_status_sets_expired_at(patched_boto3): + fake_client, _ = patched_boto3 + fake_client.get_model_invocation_job.return_value = _fake_boto3_response( + status="Expired" + ) + + batch = BedrockBatchesHandler._handle_model_invocation_job_status(batch_id=JOB_ARN) + + assert batch.status == "expired" + assert batch.expired_at == int(END_TIME.timestamp()) + assert batch.completed_at is None + assert batch.failed_at is None + assert batch.cancelled_at is None + + +def test_logging_obj_pre_and_post_call_invoked(patched_boto3): + """`pre_call` / `post_call` get called with sensible payloads when a + `logging_obj` is supplied.""" + _, _ = patched_boto3 + logging_obj = MagicMock() + + BedrockBatchesHandler._handle_model_invocation_job_status( + batch_id=JOB_ARN, logging_obj=logging_obj + ) + + logging_obj.pre_call.assert_called_once() + logging_obj.post_call.assert_called_once() + + pre_kwargs = logging_obj.pre_call.call_args.kwargs + assert pre_kwargs["input"] == JOB_ARN + assert pre_kwargs["additional_args"]["complete_input_dict"] == { + "jobIdentifier": JOB_ARN + } + # Logged URL must use the bare job id, not the full ARN, so it doesn't + # double the `model-invocation-job/` segment or embed colons in the path. + assert pre_kwargs["additional_args"]["api_base"] == ( + f"https://bedrock.us-west-2.amazonaws.com/model-invocation-job/{JOB_ID}" + ) + + post_kwargs = logging_obj.post_call.call_args.kwargs + assert post_kwargs["input"] == JOB_ARN + assert post_kwargs["original_response"]["jobArn"] == JOB_ARN + + +def test_missing_boto3_raises_helpful_import_error(): + """If boto3 isn't installed we should raise a clear, actionable + ImportError rather than letting a NameError escape.""" + real_import = ( + __builtins__["__import__"] + if isinstance(__builtins__, dict) + else __builtins__.__import__ + ) + + def fake_import(name, *args, **kwargs): + if name == "boto3": + raise ImportError("No module named 'boto3'") + return real_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=fake_import): + with pytest.raises(ImportError, match="pip install boto3"): + BedrockBatchesHandler._handle_model_invocation_job_status(batch_id=JOB_ARN) + + +def test_logging_url_uses_bare_id_when_only_id_passed(patched_boto3): + """If the caller passes just the trailing job id (also valid for + `GetModelInvocationJob`), the logged URL should use it as-is.""" + _, _ = patched_boto3 + logging_obj = MagicMock() + + BedrockBatchesHandler._handle_model_invocation_job_status( + batch_id=JOB_ID, aws_region_name="us-west-2", logging_obj=logging_obj + ) + + pre_kwargs = logging_obj.pre_call.call_args.kwargs + assert pre_kwargs["additional_args"]["api_base"] == ( + f"https://bedrock.us-west-2.amazonaws.com/model-invocation-job/{JOB_ID}" + ) diff --git a/tests/test_litellm/test_retrieve_batch_bedrock_dispatch.py b/tests/test_litellm/test_retrieve_batch_bedrock_dispatch.py new file mode 100644 index 00000000000..9df18a9f0f0 --- /dev/null +++ b/tests/test_litellm/test_retrieve_batch_bedrock_dispatch.py @@ -0,0 +1,162 @@ +"""Cover the Bedrock-ARN dispatch in ``litellm.batches.main.retrieve_batch``. + +The dispatch picks one of two Bedrock handlers depending on the ARN +family in ``batch_id``: + +* ``:async-invoke/`` -> ``_handle_async_invoke_status`` (data plane) +* ``:model-invocation-job/`` -> ``_handle_model_invocation_job_status`` + (control plane, added in this PR) + +Anything else falls through to the generic ``provider_config`` retrieve +flow. We mock the two handlers so the tests don't hit AWS — the focus +here is purely the dispatch logic that lives in ``main.py``. +""" + +from __future__ import annotations + +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm # noqa: E402 + +ASYNC_INVOKE_ARN = "arn:aws:bedrock:us-west-2:123456789012:async-invoke/abc123def456" +MIJ_ARN = "arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/abc1234567" + + +@pytest.fixture +def mock_handlers(): + """Patch both Bedrock retrieve handlers and yield the mocks. + + We patch at the import site (litellm.batches.main) rather than the + definition site so the ``BedrockBatchesHandler`` reference inside + ``retrieve_batch`` resolves to our mocks. + """ + fake_batch = MagicMock(name="LiteLLMBatch") + with ( + patch( + "litellm.batches.main.BedrockBatchesHandler._handle_async_invoke_status", + return_value=fake_batch, + ) as async_invoke, + patch( + "litellm.batches.main.BedrockBatchesHandler._handle_model_invocation_job_status", + return_value=fake_batch, + ) as mij, + ): + yield async_invoke, mij, fake_batch + + +def test_async_invoke_arn_routes_to_async_invoke_handler(mock_handlers): + """``:async-invoke/`` ARNs go to the data-plane handler.""" + async_invoke, mij, fake_batch = mock_handlers + + result = litellm.retrieve_batch( + batch_id=ASYNC_INVOKE_ARN, + custom_llm_provider="bedrock", + aws_region_name="us-west-2", + ) + + assert result is fake_batch + async_invoke.assert_called_once() + mij.assert_not_called() + call_kwargs = async_invoke.call_args.kwargs + assert call_kwargs["batch_id"] == ASYNC_INVOKE_ARN + assert call_kwargs["aws_region_name"] == "us-west-2" + # Region must be stripped from the forwarded kwargs to avoid TypeError + # (it's already an explicit positional/keyword arg). + assert "aws_region_name" not in { + k + for k in call_kwargs + if k not in {"batch_id", "aws_region_name", "logging_obj"} + } + + +def test_async_invoke_arn_falls_back_to_default_region_when_unset(mock_handlers): + """If no ``aws_region_name`` is passed, the data-plane handler defaults + to ``us-east-1`` (preserving prior behavior on this branch).""" + async_invoke, _mij, _ = mock_handlers + + litellm.retrieve_batch( + batch_id=ASYNC_INVOKE_ARN, + custom_llm_provider="bedrock", + ) + + async_invoke.assert_called_once() + assert async_invoke.call_args.kwargs["aws_region_name"] == "us-east-1" + + +def test_model_invocation_job_arn_routes_to_mij_handler(mock_handlers): + """``:model-invocation-job/`` ARNs go to the new control-plane handler.""" + _async_invoke, mij, fake_batch = mock_handlers + + result = litellm.retrieve_batch( + batch_id=MIJ_ARN, + custom_llm_provider="bedrock", + aws_region_name="us-west-2", + ) + + assert result is fake_batch + mij.assert_called_once() + _async_invoke.assert_not_called() + call_kwargs = mij.call_args.kwargs + assert call_kwargs["batch_id"] == MIJ_ARN + assert call_kwargs["aws_region_name"] == "us-west-2" + + +def test_model_invocation_job_arn_with_no_region_passes_none(mock_handlers): + """The MIJ handler is responsible for sniffing region from the ARN + when none is explicitly provided. Dispatch must forward ``None`` + rather than substituting a default — otherwise per-region jobs in + other AWS regions would silently route to ``us-east-1``.""" + _async_invoke, mij, _ = mock_handlers + + litellm.retrieve_batch( + batch_id=MIJ_ARN, + custom_llm_provider="bedrock", + ) + + mij.assert_called_once() + assert mij.call_args.kwargs["aws_region_name"] is None + + +def test_unrelated_bedrock_arn_falls_through_to_provider_config(mock_handlers): + """Bedrock ARNs that aren't async-invoke or model-invocation-job + must NOT hit either special handler — they should fall through to + the existing generic provider_config path. We don't fully exercise + that path here (it requires a real provider config); we just assert + neither special handler is invoked.""" + async_invoke, mij, _ = mock_handlers + + # Use a plausible-but-unsupported Bedrock ARN family. + unrelated_arn = "arn:aws:bedrock:us-west-2:123456789012:provisioned-model/xyz" + + with pytest.raises(Exception): + # Will raise because no provider_config exists for this path — + # that's fine, we just need to assert neither bedrock handler ran + # before the failure. + litellm.retrieve_batch( + batch_id=unrelated_arn, + custom_llm_provider="bedrock", + ) + + async_invoke.assert_not_called() + mij.assert_not_called() + + +def test_non_bedrock_id_skips_bedrock_dispatch_entirely(mock_handlers): + """Plain (non-ARN) batch ids must not even enter the Bedrock dispatch + block — they belong to other providers' retrieve flows.""" + async_invoke, mij, _ = mock_handlers + + with pytest.raises(Exception): + litellm.retrieve_batch( + batch_id="batch_abc123", + custom_llm_provider="openai", + ) + + async_invoke.assert_not_called() + mij.assert_not_called() From 9ac40925362a03804bf7c6ddc225f6a3611716a9 Mon Sep 17 00:00:00 2001 From: "oss-pr-review-agent-shin[bot]" <281797381+oss-pr-review-agent-shin[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 14:49:38 -0700 Subject: [PATCH 75/85] =?UTF-8?q?[litellm-agent]=20Staging=20=E2=86=92=20l?= =?UTF-8?q?itellm=5Finternal=5Fstaging=20(5/11/2026)=20(#27677)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Revert "feat(mavvrik): add Mavvrik integration for automatic LLM spend export…" (#27672) This reverts commit cf6fd9d87816ca37d37472c104b8652552cce3f2. * fix(proxy): update database connection timeout handling (#27507) Squash-merged by litellm-agent from harish-berri's PR. --------- Co-authored-by: Krrish Dholakia Co-authored-by: harish-berri --- litellm/proxy/proxy_cli.py | 16 ++-- tests/test_litellm/proxy/test_proxy_cli.py | 98 +++++++++++++++++++++- 2 files changed, 108 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 6359b48654b..5fc8c44b2d8 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -805,7 +805,8 @@ def run_server( # noqa: PLR0915 ) db_connection_pool_limit = 100 - db_connection_timeout = 60 + # Starts optional due to config fallback checks; guaranteed non-None before use. + db_connection_timeout: Optional[Union[int, float]] = 60 general_settings = {} ### GET DB TOKEN FOR IAM AUTH ### @@ -914,10 +915,15 @@ def run_server( # noqa: PLR0915 "database_connection_pool_limit", LiteLLMDatabaseConnectionPool.database_connection_pool_limit.value, ) - db_connection_timeout = general_settings.get( - "database_connection_pool_timeout", - LiteLLMDatabaseConnectionPool.database_connection_pool_timeout.value, - ) + db_connection_timeout = general_settings.get("database_connection_timeout") + if db_connection_timeout is None: + db_connection_timeout = general_settings.get( + "database_connection_pool_timeout" + ) + if db_connection_timeout is None: + db_connection_timeout = ( + LiteLLMDatabaseConnectionPool.database_connection_pool_timeout.value + ) if database_url and database_url.startswith("os.environ/"): original_dir = os.getcwd() # set the working directory to where this script is diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 59b43330a25..327200a6a95 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -1,6 +1,6 @@ import os import sys -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -387,6 +387,102 @@ class TestProxyInitializationHelpers: ), f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() + @pytest.mark.parametrize( + "timeout_config,expected_timeout", + [ + ({"database_connection_timeout": 30}, 30), + ({"database_connection_pool_timeout": 45}, 45), + ( + { + "database_connection_timeout": 30, + "database_connection_pool_timeout": 45, + }, + 30, + ), + ], + ) + @patch("subprocess.run") + @patch("atexit.register") + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch( + "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False + ) + def test_db_timeout_settings_are_forwarded_to_pool_timeout( + self, + mock_should_update, + mock_setup_db, + mock_atexit_register, + mock_subprocess_run, + timeout_config, + expected_timeout, + ): + from click.testing import CliRunner + + from litellm.proxy.proxy_cli import run_server + + runner = CliRunner() + mock_subprocess_run.return_value = MagicMock(returncode=0) + + mock_proxy_module = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + mock_proxy_module.ProxyConfig.return_value.get_config = AsyncMock( + return_value={ + "general_settings": { + "database_url": "postgresql://test:test@localhost:5432/test", + "database_connection_pool_limit": 5, + **timeout_config, + } + } + ) + + clean_env = { + k: v + for k, v in os.environ.items() + if k not in ("DATABASE_URL", "DIRECT_URL") + } + + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.dict( + "sys.modules", + { + "proxy_server": mock_proxy_module, + "litellm.proxy.proxy_server": mock_proxy_module, + }, + ), + patch( + "litellm.proxy.proxy_cli.ProxyInitializationHelpers._get_default_unvicorn_init_args" + ) as mock_get_args, + patch( + "litellm.proxy.proxy_cli.append_query_params", + side_effect=lambda url, params: ( + f"{url}?connection_limit={params['connection_limit']}&pool_timeout={params['pool_timeout']}" + ), + ) as mock_append_query_params, + ): + mock_get_args.return_value = { + "app": "litellm.proxy.proxy_server:app", + "host": "localhost", + "port": 8000, + } + + result = runner.invoke( + run_server, + ["--local", "--config", "test-config.yaml", "--skip_server_startup"], + ) + + assert ( + result.exit_code == 0 + ), f"exit_code={result.exit_code}, output={result.output}" + mock_append_query_params.assert_called() + appended_params = mock_append_query_params.call_args.args[1] + assert appended_params["connection_limit"] == 5 + assert appended_params["pool_timeout"] == expected_timeout + @patch("uvicorn.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") From be84d5cd7df5d9a727c46a04ac3c48f8ae04c989 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 11 May 2026 15:19:57 -0700 Subject: [PATCH 76/85] ci: add manually-triggered mutation testing workflow (#27576) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: add manually-triggered mutation testing smoke workflow Adds a workflow_dispatch-only GitHub Actions workflow that runs mutmut against a single source/test pair (router_settings_endpoints) to validate the tooling end-to-end before scaling. The workflow reinstalls litellm non-editable so the mutants/ sandbox is not shadowed by the editable .pth on sys.path, and sets PYTHONPATH so the trampolined sandbox copy wins over site-packages. mutmut itself is pulled in via uv run --with so it does not appear in uv.lock or affect the shared dev environment. Includes a temporary push: trigger scoped to this branch so we can iterate before the workflow file lands on the default branch — to be removed before merging (workflow_dispatch only requires the file on the default branch to surface the manual trigger button). * ci(mutation): disable rerun and xdist plugins for mutmut runs mutmut's in-process pytest.main() call hits `INTERNALERROR: no option named 'filtered_exceptions'` from pytest-retry's pytest_configure hook. Reruns are also wrong for mutation testing — a "failed" mutant test that gets retried would mask which mutants are killed vs. survive. Disable retry, rerunfailures, and xdist via pytest_add_cli_args in [tool.mutmut]. * ci(mutation): uninstall pytest-retry before mutmut runs `-p no:retry` (and similar names) didn't match pytest-retry's entry-point name, so the plugin still loaded and crashed during mutmut's "Running clean tests" phase. Uninstalling the package is surgical and doesn't depend on guessing the entry-point name. * ci(mutation): emit per-survivor diffs to run-page summary + artifact The previous artifact only contained `mutmut results` text (which in mutmut 3.x lists survivor names but not the actual mutations). Adds: - `mutmut export-cicd-stats` to produce mutmut-cicd-stats.json with the killed/survived/total scoreboard. - `mutmut show ` per surviving mutant to capture each mutation as a unified diff. - A `mutmut-report.md` that combines summary + run-progress tail + per-survivor diffs, written to both the artifact and $GITHUB_STEP_SUMMARY (visible on the run page, no download needed). - Corrected artifact paths: stats files live under mutants/, not the project root. - The trampolined source file from the sandbox so survivors can be inspected even outside `mutmut show`. * ci(mutation): document intended manual weekly cadence in trigger comment * ci(mutation): generate ACH-style report with embedded function bodies Replaces the inline bash markdown generation with a Python script that: - Groups survivors by function (one section per function, function body shown once per section, surviving mutants nested as subsections) - Embeds each enclosing function's source via Python AST (so the agent has full context, not just a 3-line `mutmut show` diff) - Inlines the existing test file(s) listed in [tool.mutmut].tests_dir - Writes an ACH-style task description at the bottom following the prompt template from arXiv 2501.12862 Output goes to mutation-report.md (artifact) and the head of the file is appended to $GITHUB_STEP_SUMMARY for at-a-glance visibility. * fix(mutation report): correctly parse function names with leading underscores mutmut's mutant-name prefix is x_ (single underscore), so a function named _foo produces mutants x__foo__mutmut_N. The previous regex \.x__(.+)__mutmut_ ate the function's leading underscore as part of the prefix. Changed to \.x_(.+)__mutmut_ so leading underscores are preserved in the captured function name; verified for normal, leading- underscore, and dunder-method names. * feat(mutation report): full Meta ACH-style rendering with MUTANT delimiters For each surviving mutant, parse the mutmut sandbox trampoline file and render the mutated function as it appears in the source — with the differing lines wrapped in `# MUTANT START` / `# MUTANT END` comments, matching the format from Meta's ACH paper (arXiv 2501.12862, Table 1). Renames the function header back to its original name so the agent sees the function as it would appear in the file. Falls back to the unified diff if the trampoline lookup fails. Handles replace, insert, and delete diff ops; uses difflib's SequenceMatcher to find the differing line ranges. The unified diff is preserved in a collapsible
block as secondary context. * ci(mutation): scope to whole management_endpoints folder, drop temp push trigger Final scope before merge: - paths_to_mutate / tests_dir broadened from one file to the entire management_endpoints source/test folders - Trigger is now `workflow_dispatch` only — the temporary push: block used during workflow iteration is removed - timeout-minutes bumped from 60 to 350 (just under the GH-hosted job cap of 360); whole-folder mutation against ~15 files / ~7.5k LOC can take a few hours - Artifact path for the trampoline files glob-expanded to cover all files under mutants/litellm/proxy/management_endpoints/ * fix(mutation report): warn when multiple functions in a file share a name Addresses the Greptile review concern: ast.walk's first-match-wins behavior could embed the wrong function body when a file defines the same name in multiple places (e.g., a module-level helper and a class method). mutmut's mutant identifier does not carry class context, so we can't always determine which definition was mutated. find_function_in_file now returns the start line of every matching definition; render() surfaces a "Note: N functions named X" warning in the report when there is more than one match. The first match is still embedded as the body — the warning tells the reader to verify manually instead of silently using the wrong context. Smoke-tested against the existing artifact: single-match files render unchanged. * Fix mutation report anchors * Fix mutation report TOC anchors --------- Co-authored-by: Cursor Agent --- .github/workflows/mutation-test.yml | 131 +++++++++ pyproject.toml | 27 ++ scripts/mutation_report.py | 423 ++++++++++++++++++++++++++++ 3 files changed, 581 insertions(+) create mode 100644 .github/workflows/mutation-test.yml create mode 100644 scripts/mutation_report.py diff --git a/.github/workflows/mutation-test.yml b/.github/workflows/mutation-test.yml new file mode 100644 index 00000000000..8094ca57467 --- /dev/null +++ b/.github/workflows/mutation-test.yml @@ -0,0 +1,131 @@ +name: "Mutation Test (manual)" + +# Manually-triggered mutation testing. Runs mutmut against the scope +# configured in [tool.mutmut] in pyproject.toml (currently the +# litellm/proxy/management_endpoints/ folder). Intended cadence is roughly +# weekly — clicked from the Actions tab when someone wants a fresh report. +# +# Uploads a structured `mutation-report.md` (Meta ACH-style: original + +# mutated function with `# MUTANT START`/`# MUTANT END` delimiters + the +# existing tests + a task instruction) as a workflow artifact. Failures +# do not block anything because nothing depends on this workflow. + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: mutation-test-${{ github.ref }} + cancel-in-progress: true + +jobs: + mutation: + name: Run mutmut + runs-on: ubuntu-latest + # Whole-folder mutation against ~15 files / ~7.5k LOC can take hours. + # 350 minutes is just under the GitHub-hosted job cap of 360 minutes. + timeout-minutes: 350 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + + - name: Cache uv dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/uv + .venv + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv- + + - name: Install dependencies + run: | + uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + # mutmut 3.x runs tests inside a `mutants/` sandbox where it injects + # mutation trampolines. uv installs the project as editable by default, + # which puts the original source dir on sys.path via a .pth file and + # shadows the sandbox copy — so tests would never exercise the mutated + # code. Reinstalling non-editable removes the .pth shadow. + - name: Reinstall litellm non-editable (so mutants/ is not shadowed) + run: | + uv pip uninstall litellm + uv pip install . --no-deps + + # pytest-retry's pytest_configure hook crashes with + # `INTERNALERROR: no option named 'filtered_exceptions'` when invoked + # via mutmut's in-process pytest.main() call. The entry-point name + # doesn't normalize cleanly with `-p no:`, so just remove the + # package outright. Reruns are wrong for mutation testing anyway — + # rerunning a "failed" mutant test would mask which mutants are killed. + - name: Remove pytest plugins that conflict with mutmut + run: | + uv pip uninstall pytest-retry || true + + - name: Run mutmut + env: + # Make the mutants/ sandbox win over site-packages on sys.path so the + # trampolined files are imported instead of the installed copy. + PYTHONPATH: ${{ github.workspace }}/mutants + run: | + set -o pipefail + mkdir -p mutants + uv run --no-sync --with mutmut==3.5.0 mutmut run 2>&1 | tee mutmut-run.log + + # Generate the structured report. The script embeds the enclosing + # function source for each survivor (via Python AST) and includes the + # existing test files, so an LLM agent has enough context to write + # killing tests without further file lookups. Modeled on Meta's ACH + # prompt template (arXiv 2501.12862). + - name: Generate detailed mutation report + if: always() + run: | + set +e + uv run --no-sync --with mutmut==3.5.0 mutmut export-cicd-stats > /dev/null 2>&1 + uv run --no-sync --with mutmut==3.5.0 mutmut results > mutmut-results.txt 2>&1 + uv run --no-sync python scripts/mutation_report.py + # The full report can be very long for big test files; the run-page + # summary cuts off at 1 MB. Append the head of the report (summary + # + survivor list) and link out to the artifact for the full body. + { + head -c 900000 mutation-report.md + echo "" + echo "" + echo "_Full report (with embedded function bodies and test files) is in the workflow artifact._" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload mutmut artifacts + if: always() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: mutmut-${{ github.run_id }}-${{ github.run_attempt }} + path: | + mutation-report.md + mutmut-results.txt + mutmut-run.log + mutants/mutmut-stats.json + mutants/mutmut-cicd-stats.json + mutants/litellm/proxy/management_endpoints/**/*.py + if-no-files-found: warn + retention-days: 14 diff --git a/pyproject.toml b/pyproject.toml index 5cd83148d37..65557d8a9ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -275,6 +275,33 @@ filterwarnings = [ "ignore::DeprecationWarning:pytest_asyncio.plugin", ] +[tool.mutmut] +# Mutation-testing scope. Driven by the manually-triggered workflow at +# .github/workflows/mutation-test.yml. mutmut is not part of the project's +# default install; it is pulled in via `uv run --with mutmut==` in CI. +# `also_copy = ["litellm/"]` is required because mutmut runs in a `mutants/` +# sandbox and the test conftest imports from across the litellm package. +paths_to_mutate = [ + "litellm/proxy/management_endpoints/", +] +tests_dir = [ + "tests/test_litellm/proxy/management_endpoints/", +] +also_copy = [ + "litellm/", +] +# Disable rerun/parallel plugins for mutation runs: +# - pytest-retry triggers an `INTERNALERROR: no option named 'filtered_exceptions'` +# when invoked via mutmut's in-process `pytest.main()` call. +# - rerunning a "failed" test on a mutant would mask which mutants are killed +# vs. survive, so reruns are wrong for mutation testing regardless. +# - xdist is unnecessary inside mutmut (mutmut handles its own parallelism). +pytest_add_cli_args = [ + "-p", "no:retry", + "-p", "no:rerunfailures", + "-p", "no:xdist", +] + [tool.coverage.run] source = ["litellm"] relative_files = true diff --git a/scripts/mutation_report.py b/scripts/mutation_report.py new file mode 100644 index 00000000000..a606e3f71cf --- /dev/null +++ b/scripts/mutation_report.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 +"""Generate an agent-actionable mutation testing report. + +Reads the mutmut sandbox state at `mutants/` and produces a single +`mutation-report.md` grouped by function. For each function with surviving +mutants, the report embeds the original function source (via AST), the +unified diff for each surviving mutation (via `mutmut show`), and the +existing test file(s) — followed by an ACH-style instruction asking the +reader to write tests that kill the survivors. + +Run after `mutmut run` and `mutmut export-cicd-stats`. Expects mutmut to be +invokable as `uv run --no-sync --with mutmut== mutmut `. +""" +from __future__ import annotations + +import ast +import json +import re +import subprocess +import sys +import tomllib +from collections import defaultdict +from difflib import SequenceMatcher +from pathlib import Path +from textwrap import dedent + +ROOT = Path(__file__).resolve().parent.parent +MUTMUT_INVOCATION = ["uv", "run", "--no-sync", "--with", "mutmut==3.5.0", "mutmut"] + + +def load_mutmut_config() -> dict: + with open(ROOT / "pyproject.toml", "rb") as f: + return tomllib.load(f)["tool"]["mutmut"] + + +def get_survivors() -> list[str]: + proc = subprocess.run( + [*MUTMUT_INVOCATION, "results"], capture_output=True, text=True, check=False + ) + survivors = [] + for line in proc.stdout.splitlines(): + m = re.match(r"\s*(\S+):\s*survived\s*$", line) + if m: + survivors.append(m.group(1)) + return survivors + + +def get_mutmut_show(mutant_name: str) -> str: + proc = subprocess.run( + [*MUTMUT_INVOCATION, "show", mutant_name], + capture_output=True, + text=True, + check=False, + ) + return proc.stdout.strip() or "(mutmut show produced no output)" + + +def parse_mutant_name(name: str) -> tuple[str, str, str]: + """Parse `.x___mutmut_` -> (module, function, N). + + mutmut prefixes mutated functions with `x_` (single underscore). For a + function named `foo`, mutants are `x_foo__mutmut_N`. For a function named + `_foo` (leading underscore), the mutant becomes `x__foo__mutmut_N` — so + the regex matches a single underscore after `x` and captures everything + (including any leading underscores) up to `__mutmut_`. + """ + m = re.match(r"^(.+)\.x_(.+)__mutmut_(\d+)$", name) + if not m: + return name, name, "?" + return m.group(1), m.group(2), m.group(3) + + +def function_anchor(module_path: str, function_name: str) -> str: + return re.sub(r"[^a-z0-9_-]+", "-", f"{module_path}-{function_name}".lower()).strip( + "-" + ) + + +def module_to_file(module_path: str) -> Path | None: + candidate = ROOT / Path(*module_path.split(".")).with_suffix(".py") + return candidate if candidate.exists() else None + + +def find_function_in_file( + file_path: Path, function_name: str +) -> tuple[int, int, str, list[int]] | None: + """Find a top-level or nested function by name; returns the first match. + + Returns ``(start_line, end_line, source, all_match_lines)`` or ``None``. + ``all_match_lines`` is the start line of every function (any nesting + level) in the file with this name. When ``len(all_match_lines) > 1`` the + file defines the same name in multiple places (e.g., a module-level + helper and a class method) — mutmut's mutant identifier does not carry + class context, so we can't determine which definition was mutated. + Callers surface a disambiguation note in that case. + """ + src = file_path.read_text() + tree = ast.parse(src) + matches = [ + node + for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == function_name + ] + if not matches: + return None + first = matches[0] + lines = src.splitlines() + return ( + first.lineno, + first.end_lineno, + "\n".join(lines[first.lineno - 1 : first.end_lineno]), + [m.lineno for m in matches], + ) + + +def collect_test_files(tests_dir: list[str]) -> list[Path]: + found: list[Path] = [] + for entry in tests_dir: + p = ROOT / entry + if p.is_file(): + found.append(p) + elif p.is_dir(): + found.extend(sorted(p.rglob("test_*.py"))) + return found + + +def _indent_of(line: str) -> str: + return line[: len(line) - len(line.lstrip())] + + +def render_meta_style_mutant( + module_path: str, function_name: str, mutant_num: str +) -> str | None: + """Render the mutated function with `# MUTANT START`/`# MUTANT END` delimiters. + + Reads `mutants/.py` (the trampoline file mutmut emits), finds + `x___mutmut_orig` and `x___mutmut_`, and renders the + mutated version with the lines that differ from `__mutmut_orig` wrapped + in `# MUTANT START`/`# MUTANT END` comments — the format from Meta's + ACH paper (arXiv 2501.12862, Table 1). + + The function header is rewritten to use the original function name so + the agent sees the source as it would appear in the file (rather than + mutmut's internal `x_*__mutmut_` name). + + Returns None if the trampoline file or either function cannot be found + (the caller falls back to the unified diff). + """ + trampoline = ROOT / "mutants" / Path(*module_path.split(".")).with_suffix(".py") + if not trampoline.exists(): + return None + + src = trampoline.read_text() + try: + tree = ast.parse(src) + except SyntaxError: + return None + file_lines = src.splitlines() + + orig_def = f"x_{function_name}__mutmut_orig" + mutant_def = f"x_{function_name}__mutmut_{mutant_num}" + + orig_node = mutated_node = None + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + if node.name == orig_def: + orig_node = node + elif node.name == mutant_def: + mutated_node = node + + if orig_node is None or mutated_node is None: + return None + + orig_lines = file_lines[orig_node.lineno - 1 : orig_node.end_lineno] + mutated_lines = file_lines[mutated_node.lineno - 1 : mutated_node.end_lineno] + if not orig_lines or not mutated_lines: + return None + + # Rewrite the def line to use the original (non-trampolined) function name + # so the agent sees the function as it appears in the source file. + orig_lines[0] = orig_lines[0].replace(orig_def, function_name, 1) + mutated_lines[0] = mutated_lines[0].replace(mutant_def, function_name, 1) + + matcher = SequenceMatcher(a=orig_lines, b=mutated_lines) + out: list[str] = [] + in_diff = False + + for op, i1, i2, j1, j2 in matcher.get_opcodes(): + if op == "equal": + if in_diff: + # Close the block at the indent of the line just inside it. + indent = _indent_of(out[-1]) if out else "" + out.append(f"{indent}# MUTANT END") + in_diff = False + out.extend(mutated_lines[j1:j2]) + else: + if not in_diff: + # Open the block at the indent of the first differing line. + if j1 < len(mutated_lines): + indent = _indent_of(mutated_lines[j1]) + elif i1 < len(orig_lines): + indent = _indent_of(orig_lines[i1]) + else: + indent = "" + out.append(f"{indent}# MUTANT START") + in_diff = True + if op == "delete": + # Mutation removed lines — surface what was deleted as a + # comment so the agent can see the intent of the change. + for deleted in orig_lines[i1:i2]: + indent = _indent_of(deleted) + out.append(f"{indent}# (deleted by mutation): {deleted.lstrip()}") + else: + # replace / insert: take from mutated_lines + out.extend(mutated_lines[j1:j2]) + + if in_diff: + indent = _indent_of(out[-1]) if out else "" + out.append(f"{indent}# MUTANT END") + + return "\n".join(out) + + +def render(config: dict, survivors: list[str], stats: dict | None) -> str: + by_function: dict[tuple[str, str], list[tuple[str, str]]] = defaultdict(list) + for survivor in survivors: + module_path, function_name, mutant_num = parse_mutant_name(survivor) + by_function[(module_path, function_name)].append((survivor, mutant_num)) + + out: list[str] = [] + out.append("# Mutation Test Report") + out.append("") + + out.append("## Summary") + out.append("") + if stats: + total = stats.get("total", 0) or sum( + stats.get(k, 0) + for k in ( + "killed", + "survived", + "no_tests", + "skipped", + "suspicious", + "timeout", + "segfault", + ) + ) + killed = stats.get("killed", 0) + survived = stats.get("survived", 0) + score = (killed / total * 100) if total else 0.0 + out.append(f"- Total mutants: **{total}**") + out.append(f"- Killed: **{killed}**") + out.append(f"- Survived: **{survived}**") + out.append(f"- Mutation score: **{score:.1f}%**") + for k in ("no_tests", "skipped", "suspicious", "timeout", "segfault"): + v = stats.get(k, 0) + if v: + out.append(f"- {k.replace('_', ' ').title()}: {v}") + else: + out.append(f"- Survivors found: **{len(survivors)}**") + out.append("- (mutmut-cicd-stats.json not available — full counts unavailable)") + out.append("") + + if not survivors: + out.append("**No surviving mutants — the test suite caught every mutation.**") + out.append("") + return "\n".join(out) + + out.append("## Surviving mutants by function") + out.append("") + for (module_path, function_name), items in by_function.items(): + anchor = function_anchor(module_path, function_name) + out.append( + f"- [`{function_name}`](#{anchor}) — {len(items)} mutant" + f"{'s' if len(items) != 1 else ''} ({module_path})" + ) + out.append("") + + for (module_path, function_name), items in by_function.items(): + anchor = function_anchor(module_path, function_name) + out.append(f'') + out.append(f"## `{module_path}.{function_name}`") + out.append("") + out.append(f"**Module:** `{module_path}`") + + file_path = module_to_file(module_path) + if file_path is None: + out.append("") + out.append(f"_(could not locate source file for module `{module_path}`)_") + out.append("") + else: + rel = file_path.relative_to(ROOT) + out.append(f"**File:** `{rel}`") + out.append("") + found = find_function_in_file(file_path, function_name) + if found: + start, end, fn_src, all_lines = found + out.append(f"### Original function (lines {start}-{end})") + out.append("") + if len(all_lines) > 1: + line_list = ", ".join(str(line) for line in all_lines) + out.append( + f"> **Note:** {len(all_lines)} functions named " + f"`{function_name}` are defined in this file at lines " + f"{line_list}. Showing the first match. mutmut's " + f"mutant identifier does not carry class context, so " + f"the body below may not correspond to the function " + f"that was actually mutated — verify manually before " + f"writing the killing test." + ) + out.append("") + out.append("```python") + out.append(fn_src) + out.append("```") + out.append("") + else: + out.append(f"_(could not locate `{function_name}` in {rel} via AST)_") + out.append("") + + out.append(f"### Surviving mutations ({len(items)})") + out.append("") + for i, (mutant_name, mutant_num) in enumerate(items, 1): + out.append(f"#### Mutation {i} of {len(items)} — `{mutant_name}`") + out.append("") + meta_style = render_meta_style_mutant( + module_path, function_name, mutant_num + ) + if meta_style is not None: + out.append( + "Mutated function (the bug is delimited by " + "`# MUTANT START` / `# MUTANT END`):" + ) + out.append("") + out.append("```python") + out.append(meta_style) + out.append("```") + out.append("") + out.append("
Unified diff (`mutmut show`)") + out.append("") + out.append("```diff") + out.append(get_mutmut_show(mutant_name)) + out.append("```") + out.append("") + out.append("
") + out.append("") + else: + # Fallback: trampoline file or function lookup failed. + out.append("```diff") + out.append(get_mutmut_show(mutant_name)) + out.append("```") + out.append("") + + test_files = collect_test_files(config.get("tests_dir", [])) + if test_files: + out.append("## Existing tests") + out.append("") + out.append( + "These are the test files that mutmut considered when classifying the " + "mutants above. New tests should be added here, matching existing " + "conventions, fixtures, and naming." + ) + out.append("") + for tf in test_files: + rel = tf.relative_to(ROOT) + out.append(f"### `{rel}`") + out.append("") + out.append("```python") + out.append(tf.read_text()) + out.append("```") + out.append("") + + out.append("## Task") + out.append("") + out.append( + dedent( + """\ + For each surviving mutant listed above, write a new test in the + existing test file (matching its conventions, fixtures, and naming + style) that: + + - **Fails** when the mutated version of the function is in place. + - **Passes** when the original (correct) version is in place. + + Aim for one test per surviving mutant. If multiple mutants in the + same function can be killed by a single test, that is fine — note + which mutant numbers in the test name or docstring. + + Do not modify the source file. Only add tests. + """ + ).strip() + ) + out.append("") + + return "\n".join(out) + + +def main() -> int: + config = load_mutmut_config() + + stats_file = ROOT / "mutants" / "mutmut-cicd-stats.json" + stats: dict | None = None + if stats_file.exists(): + try: + stats = json.loads(stats_file.read_text()) + except json.JSONDecodeError as exc: + print(f"warning: could not parse {stats_file}: {exc}", file=sys.stderr) + + survivors = get_survivors() + report = render(config, survivors, stats) + + out_path = ROOT / "mutation-report.md" + out_path.write_text(report) + print( + f"Wrote {out_path} ({len(survivors)} survivor" + f"{'s' if len(survivors) != 1 else ''}, {len(report)} chars)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 473cfca969a973b0addb1847bec1805c1c5006de Mon Sep 17 00:00:00 2001 From: oss-agent-shin Date: Mon, 11 May 2026 15:50:54 -0700 Subject: [PATCH 77/85] Add Bedrock Claude Platform route (#27678) * Add Claude Platform AWS Bedrock route Co-authored-by: ishaan-berri * Use Bedrock Claude Platform route Co-authored-by: ishaan-berri * Move Claude Platform route under Bedrock Co-authored-by: ishaan-berri * Split Claude Platform messages config Co-authored-by: ishaan-berri * Centralize Claude Platform Bedrock route Co-authored-by: ishaan-berri * Address Claude Platform review feedback Co-authored-by: ishaan-berri --------- Co-authored-by: oss-agent-shin <279349115+oss-agent-shin@users.noreply.github.com> Co-authored-by: ishaan-berri --- litellm/__init__.py | 6 + litellm/_lazy_imports_registry.py | 10 + litellm/llms/bedrock/base_aws_llm.py | 8 +- .../llms/bedrock/claude_platform/__init__.py | 8 + .../bedrock/claude_platform/common_utils.py | 107 ++++++ .../messages_transformation.py | 71 ++++ .../bedrock/claude_platform/transformation.py | 94 ++++++ litellm/llms/bedrock/common_utils.py | 43 ++- litellm/main.py | 28 +- litellm/utils.py | 4 + .../bedrock/test_claude_platform_provider.py | 312 ++++++++++++++++++ 11 files changed, 688 insertions(+), 3 deletions(-) create mode 100644 litellm/llms/bedrock/claude_platform/__init__.py create mode 100644 litellm/llms/bedrock/claude_platform/common_utils.py create mode 100644 litellm/llms/bedrock/claude_platform/messages_transformation.py create mode 100644 litellm/llms/bedrock/claude_platform/transformation.py create mode 100644 tests/test_litellm/llms/bedrock/test_claude_platform_provider.py diff --git a/litellm/__init__.py b/litellm/__init__.py index fd3d47ec154..e1b367fb234 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1426,6 +1426,12 @@ if TYPE_CHECKING: ) from .llms.datarobot.chat.transformation import DataRobotConfig as DataRobotConfig from .llms.anthropic.chat.transformation import AnthropicConfig as AnthropicConfig + from .llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig as BedrockClaudePlatformConfig, + ) + from .llms.bedrock.claude_platform.messages_transformation import ( + BedrockClaudePlatformMessagesConfig as BedrockClaudePlatformMessagesConfig, + ) from .llms.anthropic.completion.transformation import ( AnthropicTextConfig as AnthropicTextConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 119e62a5b38..3531e8d96b9 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -131,6 +131,7 @@ LLM_CONFIG_NAMES = ( "OpenrouterConfig", "DataRobotConfig", "AnthropicConfig", + "BedrockClaudePlatformConfig", "AnthropicTextConfig", "GroqSTTConfig", "TritonConfig", @@ -170,6 +171,7 @@ LLM_CONFIG_NAMES = ( "SagemakerNovaConfig", "CohereChatConfig", "AnthropicMessagesConfig", + "BedrockClaudePlatformMessagesConfig", "AmazonAnthropicClaudeMessagesConfig", "AmazonMantleMessagesConfig", "TogetherAIConfig", @@ -610,6 +612,10 @@ _LLM_CONFIGS_IMPORT_MAP = { "OpenrouterConfig": (".llms.openrouter.chat.transformation", "OpenrouterConfig"), "DataRobotConfig": (".llms.datarobot.chat.transformation", "DataRobotConfig"), "AnthropicConfig": (".llms.anthropic.chat.transformation", "AnthropicConfig"), + "BedrockClaudePlatformConfig": ( + ".llms.bedrock.claude_platform.transformation", + "BedrockClaudePlatformConfig", + ), "AnthropicTextConfig": ( ".llms.anthropic.completion.transformation", "AnthropicTextConfig", @@ -712,6 +718,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.anthropic.experimental_pass_through.messages.transformation", "AnthropicMessagesConfig", ), + "BedrockClaudePlatformMessagesConfig": ( + ".llms.bedrock.claude_platform.messages_transformation", + "BedrockClaudePlatformMessagesConfig", + ), "AmazonAnthropicClaudeMessagesConfig": ( ".llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation", "AmazonAnthropicClaudeMessagesConfig", diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index 0885775932c..9dd2b055a12 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -1428,7 +1428,13 @@ class BaseAWSLLM: def _sign_request( self, - service_name: Literal["bedrock", "sagemaker", "bedrock-agentcore", "s3vectors"], + service_name: Literal[ + "bedrock", + "sagemaker", + "bedrock-agentcore", + "s3vectors", + "aws-external-anthropic", + ], headers: dict, optional_params: dict, request_data: dict, diff --git a/litellm/llms/bedrock/claude_platform/__init__.py b/litellm/llms/bedrock/claude_platform/__init__.py new file mode 100644 index 00000000000..88d4e9783c7 --- /dev/null +++ b/litellm/llms/bedrock/claude_platform/__init__.py @@ -0,0 +1,8 @@ +from .transformation import ( + BedrockClaudePlatformConfig, +) +from .messages_transformation import ( + BedrockClaudePlatformMessagesConfig, +) + +__all__ = ["BedrockClaudePlatformConfig", "BedrockClaudePlatformMessagesConfig"] diff --git a/litellm/llms/bedrock/claude_platform/common_utils.py b/litellm/llms/bedrock/claude_platform/common_utils.py new file mode 100644 index 00000000000..121221518c8 --- /dev/null +++ b/litellm/llms/bedrock/claude_platform/common_utils.py @@ -0,0 +1,107 @@ +from typing import Literal, Optional, Tuple + +import litellm +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.secret_managers.main import get_secret_str + + +CLAUDE_PLATFORM_SERVICE_NAME: Literal["aws-external-anthropic"] = ( + "aws-external-anthropic" +) +CLAUDE_PLATFORM_BEDROCK_ROUTE = "claude_platform/" + + +def strip_claude_platform_route(model: str) -> str: + if model.startswith(CLAUDE_PLATFORM_BEDROCK_ROUTE): + return model.replace(CLAUDE_PLATFORM_BEDROCK_ROUTE, "", 1) + return model + + +class BedrockClaudePlatformMixin(BaseAWSLLM): + @staticmethod + def _get_workspace_id(optional_params: dict, litellm_params: dict) -> Optional[str]: + workspace_id = ( + optional_params.get("workspace_id") + or litellm_params.get("workspace_id") + or optional_params.get("aws_workspace_id") + or litellm_params.get("aws_workspace_id") + or optional_params.get("anthropic-workspace-id") + or litellm_params.get("anthropic-workspace-id") + ) + if workspace_id is None: + workspace_id = optional_params.get( + "anthropic_workspace_id" + ) or litellm_params.get("anthropic_workspace_id") + if workspace_id is not None: + return str(workspace_id) + return get_secret_str("ANTHROPIC_AWS_WORKSPACE_ID") or get_secret_str( + "ANTHROPIC_WORKSPACE_ID" + ) + + def _get_required_aws_region_name(self, optional_params: dict) -> str: + aws_region_name = ( + optional_params.get("aws_region_name") + or get_secret_str("AWS_REGION_NAME") + or get_secret_str("AWS_REGION") + or get_secret_str("AWS_DEFAULT_REGION") + ) + if aws_region_name is None: + raise litellm.AuthenticationError( + message=( + "Missing AWS region for Claude Platform on AWS. Pass " + "`aws_region_name` or set a standard AWS region environment value." + ), + llm_provider="bedrock", + model="", + ) + self._validate_aws_region_name(str(aws_region_name)) + return str(aws_region_name) + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + api_base = ( + api_base + or litellm.api_base + or get_secret_str("ANTHROPIC_AWS_BASE_URL") + or get_secret_str("ANTHROPIC_AWS_API_BASE") + ) + if api_base is None: + aws_region_name = self._get_required_aws_region_name(optional_params) + api_base = ( + f"https://{CLAUDE_PLATFORM_SERVICE_NAME}.{aws_region_name}.api.aws" + ) + if not api_base.endswith("/v1/messages"): + api_base = f"{api_base.rstrip('/')}/v1/messages" + return api_base + + def sign_request( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + ) -> Tuple[dict, Optional[bytes]]: + if api_key or get_secret_str("ANTHROPIC_AWS_API_KEY"): + return headers, None + + return self._sign_request( + service_name=CLAUDE_PLATFORM_SERVICE_NAME, + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + model=model, + stream=stream, + fake_stream=fake_stream, + ) diff --git a/litellm/llms/bedrock/claude_platform/messages_transformation.py b/litellm/llms/bedrock/claude_platform/messages_transformation.py new file mode 100644 index 00000000000..66158196322 --- /dev/null +++ b/litellm/llms/bedrock/claude_platform/messages_transformation.py @@ -0,0 +1,71 @@ +from typing import Any, Dict, List, Optional, Tuple + +import litellm +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + DEFAULT_ANTHROPIC_API_VERSION, + AnthropicMessagesConfig, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams + +from .common_utils import BedrockClaudePlatformMixin, strip_claude_platform_route + + +class BedrockClaudePlatformMessagesConfig( + BedrockClaudePlatformMixin, AnthropicMessagesConfig +): + def validate_anthropic_messages_environment( + self, + headers: dict, + model: str, + messages: List[Any], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> Tuple[dict, Optional[str]]: + workspace_id = self._get_workspace_id(optional_params, litellm_params) + if workspace_id is None: + raise litellm.AuthenticationError( + message=( + "Missing workspace ID for Claude Platform on AWS. Pass " + "`workspace_id` or configure the provider workspace setting." + ), + llm_provider="bedrock", + model=model, + ) + + resolved_api_key = api_key or get_secret_str("ANTHROPIC_AWS_API_KEY") + headers = { + **headers, + "anthropic-version": headers.get( + "anthropic-version", DEFAULT_ANTHROPIC_API_VERSION + ), + "content-type": headers.get("content-type", "application/json"), + "anthropic-workspace-id": workspace_id, + } + if resolved_api_key and "x-api-key" not in headers: + headers["x-api-key"] = resolved_api_key + + headers = self._update_headers_with_anthropic_beta( + headers=headers, + optional_params=optional_params, + ) + + return headers, api_base + + def transform_anthropic_messages_request( + self, + model: str, + messages: List[Dict], + anthropic_messages_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + return super().transform_anthropic_messages_request( + model=strip_claude_platform_route(model), + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) diff --git a/litellm/llms/bedrock/claude_platform/transformation.py b/litellm/llms/bedrock/claude_platform/transformation.py new file mode 100644 index 00000000000..0167c457c96 --- /dev/null +++ b/litellm/llms/bedrock/claude_platform/transformation.py @@ -0,0 +1,94 @@ +from typing import Any, Dict, List, Optional + +import litellm +from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues + +from .common_utils import BedrockClaudePlatformMixin + + +class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): + """ + Bedrock Claude Platform uses Anthropic's Messages API with AWS gateway auth. + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "bedrock" + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> Dict: + workspace_id = self._get_workspace_id(optional_params, litellm_params) + if workspace_id is None: + raise litellm.AuthenticationError( + message=( + "Missing workspace ID for Claude Platform on AWS. Pass " + "`workspace_id` or configure the provider workspace setting." + ), + llm_provider="bedrock", + model=model, + ) + + api_key = api_key or get_secret_str("ANTHROPIC_AWS_API_KEY") + anthropic_headers = self.get_anthropic_headers( + api_key=api_key, + auth_token=None, + computer_tool_used=self.is_computer_tool_used( + tools=optional_params.get("tools") + ), + prompt_caching_set=self.is_cache_control_set(messages=messages), + pdf_used=self.is_pdf_used(messages=messages), + file_id_used=self.is_file_id_used(messages=messages), + mcp_server_used=self.is_mcp_server_used( + mcp_servers=optional_params.get("mcp_servers") + ), + web_search_tool_used=self.is_web_search_tool_used( + tools=optional_params.get("tools") + ), + tool_search_used=self.is_tool_search_used( + tools=optional_params.get("tools") + ), + programmatic_tool_calling_used=self.is_programmatic_tool_calling_used( + tools=optional_params.get("tools") + ), + input_examples_used=self.is_input_examples_used( + tools=optional_params.get("tools") + ), + effort_used=self.is_effort_used( + optional_params=optional_params, model=model + ), + user_anthropic_beta_headers=self._get_user_anthropic_beta_headers( + anthropic_beta_header=headers.get("anthropic-beta") + ), + code_execution_tool_used=self.is_code_execution_tool_used( + tools=optional_params.get("tools") + ), + container_with_skills_used=self.is_container_with_skills_used( + optional_params=optional_params + ), + ) + anthropic_headers["anthropic-workspace-id"] = workspace_id + return {**headers, **anthropic_headers} + + def get_model_response_iterator( + self, + streaming_response: Any, + sync_stream: bool, + json_mode: Optional[bool] = False, + ) -> Any: + from litellm.llms.anthropic.chat.handler import ModelResponseIterator + + return ModelResponseIterator( + streaming_response=streaming_response, + sync_stream=sync_stream, + json_mode=bool(json_mode), + ) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 856a525f773..0256d5d4b95 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -692,6 +692,7 @@ class BedrockModelInfo(BaseLLMModelInfo): ) -> Literal[ "converse", "invoke", + "claude_platform", "converse_like", "agent", "agentcore", @@ -706,6 +707,7 @@ class BedrockModelInfo(BaseLLMModelInfo): str, Literal[ "invoke", + "claude_platform", "converse_like", "converse", "agent", @@ -716,6 +718,7 @@ class BedrockModelInfo(BaseLLMModelInfo): ], ] = { "invoke/": "invoke", + "claude_platform/": "claude_platform", "converse_like/": "converse_like", "converse/": "converse", "agent/": "agent", @@ -753,6 +756,36 @@ class BedrockModelInfo(BaseLLMModelInfo): """ return "converse/" in model + @staticmethod + def _explicit_claude_platform_route(model: str) -> bool: + """ + Check if the model is an explicit Claude Platform on AWS route. + """ + return "claude_platform/" in model + + @staticmethod + def get_claude_platform_model(model: str) -> str: + """ + Strip the Claude Platform route prefix from a Bedrock model name. + """ + return model.replace("claude_platform/", "", 1) + + @staticmethod + def map_claude_platform_auth_params( + passed_params: dict, optional_params: dict + ) -> dict: + """ + Map Claude Platform route auth params that are not OpenAI request params. + """ + for key in ( + "workspace_id", + "aws_workspace_id", + "anthropic_workspace_id", + ): + if key in passed_params: + optional_params[key] = passed_params[key] + return optional_params + @staticmethod def _explicit_invoke_route(model: str) -> bool: """ @@ -815,6 +848,12 @@ class BedrockModelInfo(BaseLLMModelInfo): All other routes should return None since they will go through litellm.completion """ + ######################################################### + # Claude Platform route uses Anthropic Messages API via the AWS gateway. + ######################################################### + if BedrockModelInfo._explicit_claude_platform_route(model): + return litellm.BedrockClaudePlatformMessagesConfig() + ######################################################### # Converse routes should go through litellm.completion() if BedrockModelInfo._explicit_converse_route(model): @@ -860,7 +899,9 @@ def get_bedrock_chat_config(model: str): base_model = BedrockModelInfo.get_base_model(model) # Handle explicit routes first - if bedrock_route == "converse" or bedrock_route == "converse_like": + if bedrock_route == "claude_platform": + return litellm.BedrockClaudePlatformConfig() + elif bedrock_route == "converse" or bedrock_route == "converse_like": return litellm.AmazonConverseConfig() elif bedrock_route == "openai": return litellm.AmazonBedrockOpenAIConfig() diff --git a/litellm/main.py b/litellm/main.py index 29a9ffc84f8..52a256fdd05 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3858,7 +3858,33 @@ def completion( # type: ignore # noqa: PLR0915 ) bedrock_route = BedrockModelInfo.get_bedrock_route(model) - if bedrock_route == "converse": + if bedrock_route == "claude_platform": + provider_config = ProviderConfigManager.get_provider_chat_config( + model=model, + provider=LlmProviders.BEDROCK, + ) + model = BedrockModelInfo.get_claude_platform_model(model) + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="bedrock", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + provider_config=provider_config, + ) + return response + elif bedrock_route == "converse": model = model.replace("converse/", "") response = bedrock_converse_chat_completion.completion( model=model, diff --git a/litellm/utils.py b/litellm/utils.py index 80452e533b6..da80e4ae164 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4407,6 +4407,10 @@ def get_optional_params( # noqa: PLR0915 else False ), ) + if bedrock_route == "claude_platform": + optional_params = BedrockModelInfo.map_claude_platform_auth_params( + passed_params=passed_params, optional_params=optional_params + ) elif custom_llm_provider == "cloudflare": optional_params = litellm.CloudflareChatConfig().map_openai_params( model=model, diff --git a/tests/test_litellm/llms/bedrock/test_claude_platform_provider.py b/tests/test_litellm/llms/bedrock/test_claude_platform_provider.py new file mode 100644 index 00000000000..74cfbb265cd --- /dev/null +++ b/tests/test_litellm/llms/bedrock/test_claude_platform_provider.py @@ -0,0 +1,312 @@ +import json +from unittest.mock import patch + +import httpx +import pytest + + +def _anthropic_response(url: str) -> httpx.Response: + return httpx.Response( + status_code=200, + json={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + request=httpx.Request("POST", url), + ) + + +def _capture_request(url: str, headers: dict, data: bytes | str | None) -> dict: + raw_body = data.decode("utf-8") if isinstance(data, bytes) else data or "{}" + return { + "path": httpx.URL(url).path, + "headers": headers, + "body": json.loads(raw_body), + } + + +def test_claude_platform_builds_default_messages_url_from_region(): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + config = BedrockClaudePlatformConfig() + + assert ( + config.get_complete_url( + api_base=None, + api_key=None, + model="claude-sonnet-4-6", + optional_params={"aws_region_name": "us-west-2"}, + litellm_params={}, + ) + == "https://aws-external-anthropic.us-west-2.api.aws/v1/messages" + ) + + +def test_claude_platform_ignores_standard_anthropic_base_url(monkeypatch): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://api.anthropic.example") + monkeypatch.setenv("ANTHROPIC_API_BASE", "https://api.anthropic-api.example") + + config = BedrockClaudePlatformConfig() + + assert ( + config.get_complete_url( + api_base=None, + api_key=None, + model="claude-sonnet-4-6", + optional_params={"aws_region_name": "us-west-2"}, + litellm_params={}, + ) + == "https://aws-external-anthropic.us-west-2.api.aws/v1/messages" + ) + + +def test_claude_platform_uses_bedrock_subroute(): + import litellm + from litellm.llms.bedrock.common_utils import BedrockModelInfo + + model, provider, _, _ = litellm.get_llm_provider( + model="bedrock/claude_platform/claude-sonnet-4-6" + ) + + assert provider == "bedrock" + assert model == "claude_platform/claude-sonnet-4-6" + assert BedrockModelInfo.get_bedrock_route(model) == "claude_platform" + assert BedrockModelInfo.get_claude_platform_model(model) == "claude-sonnet-4-6" + + +def test_claude_platform_requires_workspace_header(): + from litellm import AuthenticationError + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + config = BedrockClaudePlatformConfig() + + with pytest.raises(AuthenticationError) as exc_info: + config.validate_environment( + api_key="fake-platform-key", + headers={}, + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={}, + litellm_params={}, + ) + + assert "workspace" in str(exc_info.value).lower() + + +def test_claude_platform_api_key_auth_sets_workspace_and_key_headers(): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + config = BedrockClaudePlatformConfig() + headers = config.validate_environment( + api_key="fake-platform-key", + headers={"anthropic-beta": "skills-2025-10-02"}, + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={"workspace_id": "wrkspc_test"}, + litellm_params={}, + ) + + assert headers["x-api-key"] == "fake-platform-key" + assert headers["anthropic-workspace-id"] == "wrkspc_test" + assert headers["anthropic-beta"] == "skills-2025-10-02" + + +def test_claude_platform_does_not_use_standard_anthropic_api_key(monkeypatch): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + monkeypatch.setenv("ANTHROPIC_API_KEY", "standard-anthropic-key") + + config = BedrockClaudePlatformConfig() + headers = config.validate_environment( + api_key=None, + headers={}, + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={"workspace_id": "wrkspc_test"}, + litellm_params={}, + ) + + assert "x-api-key" not in headers + + +def test_claude_platform_sigv4_signs_transformed_request_body(): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + config = BedrockClaudePlatformConfig() + request_body = { + "model": "claude-sonnet-4-6", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 10, + } + + with patch.object( + config, + "_sign_request", + return_value=({"Authorization": "signed"}, json.dumps(request_body).encode()), + ) as mock_sign_request: + headers, signed_body = config.sign_request( + headers={"anthropic-workspace-id": "wrkspc_test"}, + optional_params={"aws_region_name": "us-west-2"}, + request_data=request_body, + api_base="https://aws-external-anthropic.us-west-2.api.aws/v1/messages", + api_key=None, + model="claude-sonnet-4-6", + ) + + assert signed_body == json.dumps(request_body).encode() + assert headers["Authorization"] == "signed" + mock_sign_request.assert_called_once() + assert ( + mock_sign_request.call_args.kwargs["service_name"] == "aws-external-anthropic" + ) + assert mock_sign_request.call_args.kwargs["request_data"] == request_body + + +def test_claude_platform_standard_anthropic_api_key_does_not_skip_sigv4(monkeypatch): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + monkeypatch.setenv("ANTHROPIC_API_KEY", "standard-anthropic-key") + config = BedrockClaudePlatformConfig() + request_body = { + "model": "claude-sonnet-4-6", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 10, + } + + with patch.object( + config, + "_sign_request", + return_value=({"Authorization": "signed"}, json.dumps(request_body).encode()), + ) as mock_sign_request: + headers, signed_body = config.sign_request( + headers={"anthropic-workspace-id": "wrkspc_test"}, + optional_params={"aws_region_name": "us-west-2"}, + request_data=request_body, + api_base="https://aws-external-anthropic.us-west-2.api.aws/v1/messages", + api_key=None, + model="claude-sonnet-4-6", + ) + + assert signed_body == json.dumps(request_body).encode() + assert headers["Authorization"] == "signed" + mock_sign_request.assert_called_once() + + +def test_bedrock_claude_platform_messages_config_round_trips_native_body(): + import litellm + from litellm.types.utils import LlmProviders + + config = litellm.ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude_platform/claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ) + + assert config is not None + headers, _ = config.validate_anthropic_messages_environment( + api_key="fake-platform-key", + headers={}, + model="claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={"max_tokens": 10}, + litellm_params={"workspace_id": "wrkspc_test"}, + ) + request_body = config.transform_anthropic_messages_request( + model="claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + anthropic_messages_optional_request_params={"max_tokens": 10}, + litellm_params={}, + headers=headers, + ) + + assert headers["anthropic-workspace-id"] == "wrkspc_test" + assert headers["x-api-key"] == "fake-platform-key" + assert request_body == { + "model": "claude-sonnet-4-6", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 10, + } + + +def test_chat_completion_routes_bedrock_claude_platform_to_messages_api(): + import litellm + + requests = [] + + def mock_post(self, url, data=None, headers=None, **kwargs): + requests.append(_capture_request(url=url, headers=headers or {}, data=data)) + return _anthropic_response(url) + + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post): + response = litellm.completion( + model="bedrock/claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + api_base="https://aws-external-anthropic.us-west-2.api.aws", + api_key="fake-platform-key", + workspace_id="wrkspc_test", + ) + + assert response.choices[0].message.content == "ok" + assert len(requests) == 1 + assert requests[0]["path"] == "/v1/messages" + assert requests[0]["headers"]["x-api-key"] == "fake-platform-key" + assert requests[0]["headers"]["anthropic-workspace-id"] == "wrkspc_test" + assert requests[0]["body"]["model"] == "claude-sonnet-4-6" + + +@pytest.mark.asyncio +async def test_anthropic_messages_routes_bedrock_claude_platform_to_messages_api(): + import litellm + + requests = [] + + async def mock_post(self, url, data=None, headers=None, **kwargs): + requests.append(_capture_request(url=url, headers=headers or {}, data=data)) + return _anthropic_response(url) + + try: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new=mock_post, + ): + response = await litellm.anthropic_messages( + model="bedrock/claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + api_base="https://aws-external-anthropic.us-west-2.api.aws", + api_key="fake-platform-key", + workspace_id="wrkspc_test", + ) + finally: + await litellm.close_litellm_async_clients() + + assert response["content"][0]["text"] == "ok" + assert len(requests) == 1 + assert requests[0]["path"] == "/v1/messages" + assert requests[0]["headers"]["x-api-key"] == "fake-platform-key" + assert requests[0]["headers"]["anthropic-workspace-id"] == "wrkspc_test" + assert requests[0]["body"]["messages"] == [{"role": "user", "content": "hello"}] + assert requests[0]["body"]["max_tokens"] == 10 + assert requests[0]["body"]["model"] == "claude-sonnet-4-6" From 6de00e24b49ae309092027ea740c0b593056aa2d Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 11 May 2026 21:08:14 -0400 Subject: [PATCH 78/85] fix(ci): unbreak realtime + bedrock batch tests (#27690) * fix(tests): drop deprecated OpenAI-Beta realtime header OpenAI deprecated the 'OpenAI-Beta: realtime=v1' header; the live service now returns code 4000 invalid_beta with "Unknown beta requested: 'realtime'.". Two integration tests in tests/llm_translation/realtime/test_realtime_guardrails_openai.py hardcoded the header and started failing across all PRs. Library code is unaffected: the OpenAI realtime handler only forwards 'OpenAI-Beta: realtime=v1' upstream when the proxy *client* sends it (litellm/llms/openai/realtime/handler.py). Default proxy behavior uses the GA protocol. Connect to OpenAI without the deprecated header, and accept the GA event name 'response.output_audio_transcript.delta' alongside the beta-protocol name 'response.audio_transcript.delta' for the transcript-delta assertion. Co-authored-by: Mateo Wang * fix(logging): post_call tolerates non-JSON-serializable values post_call() did json.dumps(original_response) without default=str, so any provider passing a dict containing datetime/Decimal/etc. would raise TypeError. Bedrock batch retrieval hits this with get_model_invocation_job() responses that include datetime fields (submitTime, lastModifiedTime, endTime), failing tests/batches_tests/test_bedrock_files_and_batches.py::test_async_file_and_batch across all PRs. Pass default=str so non-serializable values fall back to str(). Co-authored-by: Mateo Wang * fix(tests): mock boto3 in bedrock retrieve batch test The test patched AsyncHTTPHandler.get, but the bedrock retrieve handler uses boto3.client('bedrock').get_model_invocation_job directly, so the real AWS call was being made on every run, failing with AccessDeniedException because the hardcoded test ARN belongs to a different AWS account. - Mock boto3.client and BedrockBatchesConfig.get_credentials so the test never touches AWS. - Use status=Completed in the mock response so output_file_id is populated (the handler intentionally leaves it None for non-completed jobs). - Assert the predicted per-job output object URI (matches what the handler actually returns) instead of the bare output prefix. Co-authored-by: Mateo Wang * docs(tests): include GA event name in guardrail-block test docstring Co-authored-by: Mateo Wang --------- Co-authored-by: Cursor Agent Co-authored-by: Mateo Wang --- litellm/litellm_core_utils/litellm_logging.py | 2 +- .../test_bedrock_files_and_batches.py | 45 +++++++++---------- .../test_realtime_guardrails_openai.py | 11 +++-- .../test_litellm_logging.py | 13 ++++++ 4 files changed, 41 insertions(+), 30 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index a815442c2f9..c73d914e6cc 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1212,7 +1212,7 @@ class Logging(LiteLLMLoggingBaseClass): # Log the exact result from the LLM API, for streaming - log the type of response received litellm.error_logs["POST_CALL"] = locals() if isinstance(original_response, dict): - original_response = json.dumps(original_response) + original_response = json.dumps(original_response, default=str) try: self.model_call_details["input"] = input self.model_call_details["api_key"] = api_key diff --git a/tests/batches_tests/test_bedrock_files_and_batches.py b/tests/batches_tests/test_bedrock_files_and_batches.py index da08f9673d4..5148ea4db91 100644 --- a/tests/batches_tests/test_bedrock_files_and_batches.py +++ b/tests/batches_tests/test_bedrock_files_and_batches.py @@ -157,16 +157,16 @@ async def test_bedrock_retrieve_batch(): """ print("Testing bedrock batch retrieval") - # Mock bedrock batch response mock_bedrock_response = { "jobArn": "arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/test-job-123", "jobName": "test-job-123", "modelId": "us.anthropic.claude-haiku-4-5-20251001-v1:0", "roleArn": "arn:aws:iam::123456789012:role/service-role/AmazonBedrockExecutionRoleForAgents_TEST", - "status": "InProgress", - "message": "Job is in progress", + "status": "Completed", + "message": "", "submitTime": "2024-01-01T12:00:00Z", "lastModifiedTime": "2024-01-01T12:30:00Z", + "endTime": "2024-01-01T13:00:00Z", "inputDataConfig": { "s3InputDataConfig": {"s3Uri": "s3://test-bucket/input/test-input.jsonl"} }, @@ -175,43 +175,38 @@ async def test_bedrock_retrieve_batch(): }, } - # Mock the HTTP response - mock_response = MagicMock() - mock_response.json.return_value = mock_bedrock_response - mock_response.status_code = 200 + mock_bedrock_client = MagicMock() + mock_bedrock_client.get_model_invocation_job.return_value = mock_bedrock_response + mock_creds = MagicMock(access_key="ak", secret_key="sk", token="tok") - # Print the mock response to debug - print("MOCK RESPONSE DATA:", mock_bedrock_response) - - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get" - ) as mock_get: - mock_response.raise_for_status.return_value = None - mock_get.return_value = mock_response - - # Test retrieve batch + with ( + patch("boto3.client", return_value=mock_bedrock_client), + patch( + "litellm.llms.bedrock.batches.transformation.BedrockBatchesConfig.get_credentials", + return_value=mock_creds, + ), + ): batch_response = await litellm.aretrieve_batch( batch_id="arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/test-job-123", custom_llm_provider="bedrock", model="us.anthropic.claude-haiku-4-5-20251001-v1:0", ) - print("MOCKED BATCH RESPONSE=", batch_response) - - # Validate the response assert ( batch_response.id == "arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/test-job-123" ) assert batch_response.object == "batch" - assert ( - batch_response.status == "in_progress" - ) # Bedrock "InProgress" maps to "in_progress" + assert batch_response.status == "completed" assert batch_response.endpoint == "/v1/chat/completions" - # Validate input and output file IDs in the final transformed response assert batch_response.input_file_id == "s3://test-bucket/input/test-input.jsonl" - assert batch_response.output_file_id == "s3://test-bucket/output/" + # Bedrock returns only the output *prefix*; the handler predicts the + # actual output object as //.out. + assert ( + batch_response.output_file_id + == "s3://test-bucket/output/test-job-123/test-input.jsonl.out" + ) def test_bedrock_batch_with_encryption_key_in_post_request(): diff --git a/tests/llm_translation/realtime/test_realtime_guardrails_openai.py b/tests/llm_translation/realtime/test_realtime_guardrails_openai.py index 884563c6e5c..170440f6b9f 100644 --- a/tests/llm_translation/realtime/test_realtime_guardrails_openai.py +++ b/tests/llm_translation/realtime/test_realtime_guardrails_openai.py @@ -104,7 +104,8 @@ async def test_text_message_blocked_by_guardrail_no_ai_response(): Send a text message containing the blocked phrase. Guardrail must: - Send error event (guardrail_violation) to client. - - Send response.audio_transcript.delta with the block message to client. + - Send response.output_audio_transcript.delta (or beta-protocol + response.audio_transcript.delta) with the block message to client. - NOT forward response.create to OpenAI (no AI response). """ import websockets @@ -119,7 +120,6 @@ async def test_text_message_blocked_by_guardrail_no_ai_response(): OPENAI_REALTIME_URL, additional_headers={ "Authorization": f"Bearer {OPENAI_API_KEY}", - "OpenAI-Beta": "realtime=v1", }, ) as backend_ws: streaming, input_queue = await _build_streaming(client_events, backend_ws) @@ -182,7 +182,11 @@ async def test_text_message_blocked_by_guardrail_no_ai_response(): transcript_deltas = [ e for e in client_events - if e.get("type") == "response.audio_transcript.delta" + if e.get("type") + in ( + "response.output_audio_transcript.delta", + "response.audio_transcript.delta", + ) ] assert ( len(transcript_deltas) >= 1 @@ -298,7 +302,6 @@ async def test_clean_text_message_passes_through_to_openai(): OPENAI_REALTIME_URL, additional_headers={ "Authorization": f"Bearer {OPENAI_API_KEY}", - "OpenAI-Beta": "realtime=v1", }, ) as backend_ws: streaming, input_queue = await _build_streaming(client_events, backend_ws) diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 1764d9c609f..e84baf5e137 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -36,6 +36,19 @@ def test_get_masked_api_base(logging_obj): assert type(masked_api_base) == str +def test_post_call_serializes_dict_with_datetime(logging_obj): + import datetime + + response = { + "status": "InProgress", + "submitTime": datetime.datetime(2026, 5, 11, 23, 49, 13, 132000), + } + logging_obj.post_call(original_response=response) + serialized = logging_obj.model_call_details["original_response"] + assert isinstance(serialized, str) + assert "2026-05-11" in serialized + + def test_sentry_sample_rate(): existing_sample_rate = os.getenv("SENTRY_API_SAMPLE_RATE") try: From f3b8aad883e502826078be4af7678c463242306d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 11 May 2026 19:10:31 -0700 Subject: [PATCH 79/85] fix(router): pin Responses API affinity to Azure resource on model-group switch When a Responses API follow-up switches model_name (e.g. gpt-5.3-codex -> gpt-5.4, or to a LiteLLM-side alias of the same Azure deployment), the router has already filtered healthy_deployments to the new group, so the originating model_id is no longer present. The encrypted_content_affinity check would log "decoded deployment not found" and fall back to the full deployment pool, where simple-shuffle could land on a different Azure resource and trip a 400 invalid_encrypted_content. Fall back to pinning by the originating deployment's encryption boundary (api_base + api_key) when the model_id miss is across model groups. The encrypted_content travels with the Azure resource, not the model_name, so any deployment on the same resource accepts it. LIT-2531 --- litellm/router.py | 2 +- .../encrypted_content_affinity_check.py | 70 +++- .../test_encrypted_content_affinity_check.py | 305 ++++++++++++++++++ 3 files changed, 373 insertions(+), 4 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 37295f1a7d2..5e30ae618c6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1673,7 +1673,7 @@ class Router: for cb in self.optional_callbacks ) if not already_registered: - ec_callback = EncryptedContentAffinityCheck() + ec_callback = EncryptedContentAffinityCheck(router=self) self.optional_callbacks.append(ec_callback) litellm.logging_callback_manager.add_litellm_callback(ec_callback) diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 3f1714ba5a5..31e589e54e6 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -36,13 +36,16 @@ Safe to enable globally: - No cache required. """ -from typing import Any, List, Optional, cast +from typing import TYPE_CHECKING, Any, List, Optional, cast from litellm._logging import verbose_router_logger from litellm.integrations.custom_logger import CustomLogger, Span from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import AllMessageValues +if TYPE_CHECKING: + from litellm.router import Router + class EncryptedContentAffinityCheck(CustomLogger): """ @@ -55,8 +58,9 @@ class EncryptedContentAffinityCheck(CustomLogger): Wired via ``Router(optional_pre_call_checks=["encrypted_content_affinity"])``. """ - def __init__(self) -> None: + def __init__(self, router: Optional["Router"] = None) -> None: super().__init__() + self.router = router # ------------------------------------------------------------------ # Helpers @@ -119,6 +123,49 @@ class EncryptedContentAffinityCheck(CustomLogger): return deployment return None + @staticmethod + def _encryption_boundary_key( + litellm_params: dict, + ) -> Optional[tuple]: + """ + ``(api_base, api_key)`` pair identifying an Azure resource. Two + deployments sharing both are interchangeable for ``encrypted_content`` + follow-ups; Azure rejects content produced by any other resource. + """ + if not isinstance(litellm_params, dict): + return None + api_base = litellm_params.get("api_base") + api_key = litellm_params.get("api_key") + if not api_base or not api_key: + return None + return (api_base, api_key) + + def _find_deployments_on_same_encryption_boundary( + self, + healthy_deployments: List[dict], + model_id: str, + ) -> List[dict]: + """ + Deployments in ``healthy_deployments`` sharing the originating + deployment's ``(api_base, api_key)``. Returns ``[]`` if router isn't + wired in, the originating deployment was removed, or no boundary match. + """ + if self.router is None: + return [] + originating = self.router.get_deployment(model_id=model_id) + if originating is None: + return [] + boundary = self._encryption_boundary_key( + originating.litellm_params.model_dump(exclude_none=True) + ) + if boundary is None: + return [] + return [ + d + for d in healthy_deployments + if self._encryption_boundary_key(d.get("litellm_params", {})) == boundary + ] + # ------------------------------------------------------------------ # Request routing (pre-call filter) # ------------------------------------------------------------------ @@ -172,8 +219,25 @@ class EncryptedContentAffinityCheck(CustomLogger): request_kwargs["_encrypted_content_affinity_pinned"] = True return [deployment] + # Follow-up switched model_name (LIT-2531): pin by Azure resource instead. + boundary_matches = self._find_deployments_on_same_encryption_boundary( + healthy_deployments=typed_healthy_deployments, + model_id=model_id, + ) + if boundary_matches: + verbose_router_logger.debug( + "EncryptedContentAffinityCheck: model_id=%s not in healthy_deployments; " + "pinning to %d deployment(s) on same encryption boundary", + model_id, + len(boundary_matches), + ) + request_kwargs["_encrypted_content_affinity_pinned"] = True + return boundary_matches + verbose_router_logger.error( - "EncryptedContentAffinityCheck: decoded deployment=%s not found in healthy_deployments", + "EncryptedContentAffinityCheck: decoded deployment=%s not found in " + "healthy_deployments and no boundary match available; falling back to " + "full deployment pool (encrypted_content may be rejected upstream)", model_id, ) return typed_healthy_deployments diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index cbc4a920245..a4faf5c37aa 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -791,3 +791,308 @@ def test_encrypted_content_wrapping_empty_string(): assert extracted_model_id == model_id assert unwrapped == original_content + + +# --------------------------------------------------------------------------- +# LIT-2531: cross-model-group fallback via encryption boundary (api_base + api_key) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_affinity_falls_back_to_same_encryption_boundary_on_model_group_switch(): + """ + LIT-2531: Client starts a session on gpt-5.3-codex, follow-up switches to + gpt-5.4 mid-chat (e.g. via Codex `model_migrations`). Affinity must pin to + the gpt-5.4 deployment on the SAME Azure resource as the originating + gpt-5.3-codex deployment -- otherwise Azure rejects the encrypted_content. + """ + first_resp = _build_mock_response( + output_items=[ + { + "type": "reasoning", + "id": "rs_encrypted_xyz", + "status": "completed", + "encrypted_content": "gAAAAABpnW_yEYmSNEyOG...", + }, + ], + response_id="resp_first", + ) + second_resp = _build_mock_response( + output_items=[ + { + "type": "message", + "id": "msg_ok", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "answer"}], + }, + ], + response_id="resp_second", + ) + + ACCOUNT_A_BASE = "https://account-a.openai.azure.com/" + ACCOUNT_A_KEY = "key-a" + ACCOUNT_B_BASE = "https://account-b.openai.azure.com/" + ACCOUNT_B_KEY = "key-b" + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-5.3-codex", + "litellm_params": { + "model": "azure/gpt-5.3-codex", + "api_base": ACCOUNT_A_BASE, + "api_key": ACCOUNT_A_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.3-codex-account-a"}, + }, + { + "model_name": "gpt-5.3-codex", + "litellm_params": { + "model": "azure/gpt-5.3-codex", + "api_base": ACCOUNT_B_BASE, + "api_key": ACCOUNT_B_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.3-codex-account-b"}, + }, + { + "model_name": "gpt-5.4", + "litellm_params": { + "model": "azure/gpt-5.4", + "api_base": ACCOUNT_A_BASE, + "api_key": ACCOUNT_A_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.4-account-a"}, + }, + { + "model_name": "gpt-5.4", + "litellm_params": { + "model": "azure/gpt-5.4", + "api_base": ACCOUNT_B_BASE, + "api_key": ACCOUNT_B_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.4-account-b"}, + }, + ], + optional_pre_call_checks=["encrypted_content_affinity"], + num_retries=0, + ) + + def first_call_picks_account_a(seq): + for d in seq: + if d["model_info"]["id"] == "gpt-5.3-codex-account-a": + return d + return seq[0] + + with patch( + "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", + new_callable=AsyncMock, + return_value=first_resp, + ), patch( + "litellm.router_strategy.simple_shuffle.random.choice", + side_effect=first_call_picks_account_a, + ): + r1 = await router.aresponses(model="gpt-5.3-codex", input="hi") + + assert r1._hidden_params["model_id"] == "gpt-5.3-codex-account-a" + encoded_id = _extract_encoded_item_id(r1) + assert encoded_id.startswith("encitem_") + + # simple_shuffle.random.choice NOT patched: prove affinity narrows the + # candidate pool to a single deployment regardless of which one shuffle picks. + with patch( + "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", + new_callable=AsyncMock, + return_value=second_resp, + ): + r2 = await router.aresponses( + model="gpt-5.4", + input=[ + { + "type": "reasoning", + "id": encoded_id, + "encrypted_content": "gAAAAABpnW_yEYmSNEyOG...", + }, + ], + ) + + assert r2._hidden_params["model_id"] == "gpt-5.4-account-a" + + +@pytest.mark.asyncio +async def test_affinity_falls_back_to_same_boundary_on_alias_switch(): + """ + LIT-2531 alias path: gpt-5.2-codex is a LiteLLM alias that points at the + same underlying Azure model as gpt-5.3-codex. Different model_name groups + in the router, so model_id-based pinning misses, but the encryption + boundary (api_base + api_key) is identical -> follow-up must still pin. + """ + first_resp = _build_mock_response( + output_items=[ + { + "type": "reasoning", + "id": "rs_alias_xyz", + "status": "completed", + "encrypted_content": "gAAAAABpnW_yEYmSNEyOG...", + }, + ], + response_id="resp_alias_first", + ) + second_resp = _build_mock_response( + output_items=[ + { + "type": "message", + "id": "msg_alias_ok", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok"}], + }, + ], + response_id="resp_alias_second", + ) + + ACCOUNT_A_BASE = "https://account-a.openai.azure.com/" + ACCOUNT_A_KEY = "key-a" + ACCOUNT_B_BASE = "https://account-b.openai.azure.com/" + ACCOUNT_B_KEY = "key-b" + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-5.3-codex", + "litellm_params": { + "model": "azure/gpt-5.3-codex", + "api_base": ACCOUNT_A_BASE, + "api_key": ACCOUNT_A_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.3-codex-account-a"}, + }, + { + "model_name": "gpt-5.3-codex", + "litellm_params": { + "model": "azure/gpt-5.3-codex", + "api_base": ACCOUNT_B_BASE, + "api_key": ACCOUNT_B_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.3-codex-account-b"}, + }, + { + "model_name": "gpt-5.2-codex", + "litellm_params": { + "model": "azure/gpt-5.3-codex", + "api_base": ACCOUNT_A_BASE, + "api_key": ACCOUNT_A_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.2-codex-account-a"}, + }, + { + "model_name": "gpt-5.2-codex", + "litellm_params": { + "model": "azure/gpt-5.3-codex", + "api_base": ACCOUNT_B_BASE, + "api_key": ACCOUNT_B_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.2-codex-account-b"}, + }, + ], + optional_pre_call_checks=["encrypted_content_affinity"], + num_retries=0, + ) + + def pick_account_a(seq): + for d in seq: + if d["model_info"]["id"] == "gpt-5.3-codex-account-a": + return d + return seq[0] + + with patch( + "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", + new_callable=AsyncMock, + return_value=first_resp, + ), patch( + "litellm.router_strategy.simple_shuffle.random.choice", + side_effect=pick_account_a, + ): + r1 = await router.aresponses(model="gpt-5.3-codex", input="hi") + + encoded_id = _extract_encoded_item_id(r1) + assert encoded_id.startswith("encitem_") + + with patch( + "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", + new_callable=AsyncMock, + return_value=second_resp, + ): + r2 = await router.aresponses( + model="gpt-5.2-codex", + input=[ + { + "type": "reasoning", + "id": encoded_id, + "encrypted_content": "gAAAAABpnW_yEYmSNEyOG...", + }, + ], + ) + + assert r2._hidden_params["model_id"] == "gpt-5.2-codex-account-a" + + +def test_boundary_fallback_no_router_ref_returns_empty(): + """ + Standalone use (no router wired in) -> the boundary lookup short-circuits + to ``[]`` instead of crashing on ``None.get_deployment``. + """ + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + check = EncryptedContentAffinityCheck(router=None) + healthy = [ + { + "model_info": {"id": "dep-1"}, + "litellm_params": {"api_base": "https://x", "api_key": "k"}, + } + ] + matches = check._find_deployments_on_same_encryption_boundary( + healthy_deployments=healthy, + model_id="dep-2", + ) + assert matches == [] + + +def test_boundary_fallback_originating_deployment_removed_returns_empty(): + """ + If the originating deployment has been removed from the router (e.g. via + /model/delete), ``router.get_deployment`` returns None and we return [] so + the caller falls back to the full healthy_deployments list. + """ + from unittest.mock import MagicMock + + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + mock_router = MagicMock() + mock_router.get_deployment.return_value = None + + check = EncryptedContentAffinityCheck(router=mock_router) + healthy = [ + { + "model_info": {"id": "dep-1"}, + "litellm_params": {"api_base": "https://x", "api_key": "k"}, + } + ] + matches = check._find_deployments_on_same_encryption_boundary( + healthy_deployments=healthy, + model_id="dep-removed", + ) + assert matches == [] + mock_router.get_deployment.assert_called_once_with(model_id="dep-removed") From 40db114a23142ad37ef8298ca43f837314426eb8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 12 May 2026 03:09:08 +0000 Subject: [PATCH 80/85] fix(router): accept Pydantic LiteLLM_Params in encryption-boundary key lookup Greptile flagged that the strict isinstance(dict) guard in _encryption_boundary_key would silently return None for any non-dict input, including a LiteLLM_Params Pydantic instance, which exposes a custom .get() method and is intended to be used dict-style in some router paths. If such an instance ever flowed into healthy_deployments, the guard would drop every candidate from boundary matching and fall through to the full deployment pool, i.e. trigger the exact invalid_encrypted_content failure this check exists to prevent. Loosen the guard to accept any object exposing a callable .get(): plain dicts (the common case) and LiteLLM_Params-style Pydantic instances. The function still returns None for non-dict-like values (None, lists, strings, ints, bare objects). Adds regression tests covering: - LiteLLM_Params Pydantic instance resolves to the same boundary tuple as an equivalent plain dict - non-dict-like values and dicts missing required fields still return None --- .../encrypted_content_affinity_check.py | 17 ++- .../test_encrypted_content_affinity_check.py | 104 +++++++++++++++--- 2 files changed, 103 insertions(+), 18 deletions(-) diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 31e589e54e6..866073c84dc 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -125,17 +125,26 @@ class EncryptedContentAffinityCheck(CustomLogger): @staticmethod def _encryption_boundary_key( - litellm_params: dict, + litellm_params: Any, ) -> Optional[tuple]: """ ``(api_base, api_key)`` pair identifying an Azure resource. Two deployments sharing both are interchangeable for ``encrypted_content`` follow-ups; Azure rejects content produced by any other resource. + + Accepts any object exposing dict-style ``.get(key, default)``: plain + dicts (the common case in ``healthy_deployments``) as well as + ``LiteLLM_Params``-style Pydantic instances, which define a custom + ``.get()``. A stricter ``isinstance(dict)`` guard would silently drop + the latter from boundary matching and fall back to the full pool — + i.e. trigger the exact ``invalid_encrypted_content`` failure this + check exists to prevent. """ - if not isinstance(litellm_params, dict): + getter = getattr(litellm_params, "get", None) + if not callable(getter): return None - api_base = litellm_params.get("api_base") - api_key = litellm_params.get("api_key") + api_base = getter("api_base") + api_key = getter("api_key") if not api_base or not api_key: return None return (api_base, api_key) diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index a4faf5c37aa..1f6848d3b5b 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -888,13 +888,16 @@ async def test_affinity_falls_back_to_same_encryption_boundary_on_model_group_sw return d return seq[0] - with patch( - "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", - new_callable=AsyncMock, - return_value=first_resp, - ), patch( - "litellm.router_strategy.simple_shuffle.random.choice", - side_effect=first_call_picks_account_a, + with ( + patch( + "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", + new_callable=AsyncMock, + return_value=first_resp, + ), + patch( + "litellm.router_strategy.simple_shuffle.random.choice", + side_effect=first_call_picks_account_a, + ), ): r1 = await router.aresponses(model="gpt-5.3-codex", input="hi") @@ -1013,13 +1016,16 @@ async def test_affinity_falls_back_to_same_boundary_on_alias_switch(): return d return seq[0] - with patch( - "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", - new_callable=AsyncMock, - return_value=first_resp, - ), patch( - "litellm.router_strategy.simple_shuffle.random.choice", - side_effect=pick_account_a, + with ( + patch( + "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", + new_callable=AsyncMock, + return_value=first_resp, + ), + patch( + "litellm.router_strategy.simple_shuffle.random.choice", + side_effect=pick_account_a, + ), ): r1 = await router.aresponses(model="gpt-5.3-codex", input="hi") @@ -1096,3 +1102,73 @@ def test_boundary_fallback_originating_deployment_removed_returns_empty(): ) assert matches == [] mock_router.get_deployment.assert_called_once_with(model_id="dep-removed") + + +def test_boundary_key_accepts_pydantic_litellm_params_instance(): + """ + Regression: ``_encryption_boundary_key`` must accept any object exposing + dict-style ``.get()`` (incl. ``LiteLLM_Params`` Pydantic instances) — not + just plain dicts. + + A stricter ``isinstance(dict)`` guard would silently return ``None`` for a + ``LiteLLM_Params`` value, drop the deployment from boundary matching, and + fall back to the full pool — which is the exact ``invalid_encrypted_content`` + failure this check exists to prevent. + """ + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + from litellm.types.router import LiteLLM_Params + + pydantic_params = LiteLLM_Params( + model="azure/gpt-5.3-codex", + api_base="https://mateo-resource.openai.azure.com", + api_key="fake-azure-resource-key-a", + ) + plain_params = { + "model": "azure/gpt-5.3-codex", + "api_base": "https://mateo-resource.openai.azure.com", + "api_key": "fake-azure-resource-key-a", + } + + pydantic_key = EncryptedContentAffinityCheck._encryption_boundary_key( + pydantic_params + ) + plain_key = EncryptedContentAffinityCheck._encryption_boundary_key(plain_params) + + assert pydantic_key is not None + assert ( + pydantic_key + == plain_key + == ( + "https://mateo-resource.openai.azure.com", + "fake-azure-resource-key-a", + ) + ) + + +def test_boundary_key_rejects_non_dict_like_inputs(): + """ + Inputs that don't expose ``.get()`` (None, lists, strings, ints) -> None. + Guards against accidentally treating a stray non-dict-like value as a + valid boundary. + """ + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + for bad in (None, [], "not a dict", 42, object()): + assert EncryptedContentAffinityCheck._encryption_boundary_key(bad) is None + + assert ( + EncryptedContentAffinityCheck._encryption_boundary_key( + {"api_base": "", "api_key": "k"} + ) + is None + ) + assert ( + EncryptedContentAffinityCheck._encryption_boundary_key( + {"api_base": "https://x"} + ) + is None + ) From aa9e7b9808af4a46d496ca2f2436edf8ed2f5d77 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 12 May 2026 09:01:43 +0530 Subject: [PATCH 81/85] feat: litellm shin agent oss staging 05 10 2026 (#27631) * fix: invalidate cached tag object on tag budget reset (#27481) (#27572) Squash-merged by litellm-agent from oss-agent-shin's PR. * chore(mcp): tighten stdio server registration paths (#27570) Squash-merged by litellm-agent from stuxf's PR. * fix(proxy): clear MCP OpenAPI mappings on server eviction; widen budget cache invalidation Evict OpenAPI tools from global_mcp_tool_registry and strip tool_name_to_mcp_server_name_mapping entries when a server leaves the runtime registry (remove_server and approval-status eviction). Invalidate user_api_key_cache for keys, orgs, and team members on budget-tier spend resets alongside tags. Co-authored-by: Cursor * fix(mcp): align update_server eviction with remove_server name fallback Document budget-reset test assertion flip (cross-pod cache staleness). Greptile: eviction now pops by server_id then server_name like remove_server; test docstring explains assert_not_awaited -> assert_any_await change. Co-authored-by: Cursor * Fix org budget cache invalidation --------- Co-authored-by: oss-agent-shin Co-authored-by: stuxf <70670632+stuxf@users.noreply.github.com> Co-authored-by: Cursor --- .../mcp_server/mcp_server_manager.py | 70 +++++++++- .../_experimental/mcp_server/tool_registry.py | 16 +++ .../proxy/common_utils/reset_budget_job.py | 35 +++-- .../mcp_management_endpoints.py | 19 +++ tests/mcp_tests/test_mcp_server.py | 3 + .../mcp_server/test_mcp_server_manager.py | 127 +++++++++++++++++- .../common_utils/test_reset_budget_job.py | 103 ++++++++++++-- .../test_mcp_management_endpoints.py | 26 ++++ 8 files changed, 370 insertions(+), 29 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 6ad731e7113..4901bc76d2f 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -599,16 +599,57 @@ class MCPServerManager: ) raise e + def _cleanup_server_tool_routing_artifacts(self, server: MCPServer) -> None: + """Drop OpenAPI global tools and name-mapping rows owned by ``server``. + + When a server leaves ``self.registry`` (eviction, ``remove_server``, etc.), + OpenAPI tools remain in ``global_mcp_tool_registry`` and + ``tool_name_to_mcp_server_name_mapping`` unless removed here. Stale + mappings make ``_get_mcp_server_from_tool_name`` resolve to a prefix that + no longer exists in the live registry. + """ + from litellm.proxy._experimental.mcp_server.tool_registry import ( + global_mcp_tool_registry, + ) + + prefix_root = normalize_server_name(get_server_prefix(server)) + if server.spec_path and prefix_root: + openapi_key_prefix = prefix_root + MCP_TOOL_PREFIX_SEPARATOR + global_mcp_tool_registry.unregister_tools_with_prefix(openapi_key_prefix) + + owned_raw: Set[str] = set() + for p in iter_known_server_prefixes(server): + if p: + owned_raw.add(p) + if server.name: + owned_raw.add(server.name) + + owned_normalized = {normalize_server_name(x) for x in owned_raw} + + stale_mapping_keys: List[str] = [] + for tool_name, mapped_server in list( + self.tool_name_to_mcp_server_name_mapping.items() + ): + if mapped_server in owned_raw: + stale_mapping_keys.append(tool_name) + elif normalize_server_name(str(mapped_server)) in owned_normalized: + stale_mapping_keys.append(tool_name) + + for key in stale_mapping_keys: + del self.tool_name_to_mcp_server_name_mapping[key] + def remove_server(self, mcp_server: LiteLLM_MCPServerTable): """ Remove a server from the registry """ - if mcp_server.server_name in self.get_registry(): - del self.registry[mcp_server.server_name] - verbose_logger.debug(f"Removed MCP Server: {mcp_server.server_name}") - elif mcp_server.server_id in self.get_registry(): - del self.registry[mcp_server.server_id] - verbose_logger.debug(f"Removed MCP Server: {mcp_server.server_id}") + evicted: Optional[MCPServer] = self.registry.pop(mcp_server.server_id, None) + if evicted is None and mcp_server.server_name: + evicted = self.registry.pop(mcp_server.server_name, None) + if evicted is not None: + verbose_logger.debug( + "Removed MCP Server: %s", mcp_server.server_id or mcp_server.server_name + ) + self._cleanup_server_tool_routing_artifacts(evicted) else: verbose_logger.warning( f"Server ID {mcp_server.server_id} not found in registry" @@ -806,6 +847,13 @@ class MCPServerManager: self.initialize_tool_name_to_mcp_server_name_mapping() async def add_server(self, mcp_server: LiteLLM_MCPServerTable): + # The runtime registry is the allowlist for tool calls and health + # probes (which spawn the underlying transport, including stdio + # subprocesses). Match the eligibility set used by the bulk DB + # filter in reload_servers_from_database() — NULL is legacy and + # "approved" is a legacy alias for "active". + if mcp_server.approval_status not in (None, "active", "approved"): + return try: if mcp_server.server_id not in self.registry: new_server = await self.build_mcp_server_from_table(mcp_server) @@ -819,6 +867,16 @@ class MCPServerManager: raise e async def update_server(self, mcp_server: LiteLLM_MCPServerTable): + # If a previously-active server has been moved out of the active + # state, evict any stale registry entry so subsequent tool calls and + # health probes can't reach it. + if mcp_server.approval_status not in (None, "active", "approved"): + evicted = self.registry.pop(mcp_server.server_id, None) + if evicted is None and mcp_server.server_name: + evicted = self.registry.pop(mcp_server.server_name, None) + if evicted is not None: + self._cleanup_server_tool_routing_artifacts(evicted) + return try: if mcp_server.server_id in self.registry: new_server = await self.build_mcp_server_from_table(mcp_server) diff --git a/litellm/proxy/_experimental/mcp_server/tool_registry.py b/litellm/proxy/_experimental/mcp_server/tool_registry.py index 58570aafadf..829be5be979 100644 --- a/litellm/proxy/_experimental/mcp_server/tool_registry.py +++ b/litellm/proxy/_experimental/mcp_server/tool_registry.py @@ -59,6 +59,22 @@ class MCPToolRegistry: ] return list(self.tools.values()) + def unregister_tools_with_prefix(self, prefix: str) -> int: + """Remove tools whose registered name starts with ``prefix``. + + Used when an OpenAPI-backed MCP server leaves the runtime registry so + stale tool handlers cannot be invoked after eviction. + """ + if not prefix: + return 0 + removed = 0 + for name in list(self.tools.keys()): + if name.startswith(prefix): + del self.tools[name] + removed += 1 + verbose_logger.debug("Unregistered MCP tool %s", name) + return removed + def convert_tools_to_mcp_sdk_tool_type( self, tools: List[MCPTool] ) -> List["MCPToolSDKTool"]: diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 71537cc62e6..52bbeaf2ad3 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -87,13 +87,13 @@ class ResetBudgetJob: async def _invalidate_user_api_key_cache_entry(cache_key: str) -> None: """Drop a stale management-cache entry so the next read fetches from DB. - Some entity types (notably tags and end-users) are not handled by - SpendCounterReseed.from_db, so when a spend counter expires the - budget check falls back to ``cached_obj.spend``. If that cached - object lingers in ``user_api_key_cache`` past a budget reset, the - stale ``.spend`` keeps the entity blocked indefinitely. Deleting - the cache entry forces the next auth-time fetch to reload the - zeroed row from Postgres. + Tags and end-users are not reseeded by ``SpendCounterReseed.from_db``; + for those, when the spend counter expires the budget check falls back + to ``cached_obj.spend``. Keys, orgs, and team memberships are reseeded + from the DB, but auth still may consult ``user_api_key_cache`` objects + whose ``.spend`` field can lag a cross-pod DB reset. Deleting the cache + entry forces the next auth-time fetch to reload the zeroed row from + Postgres. """ try: from litellm.proxy.proxy_server import user_api_key_cache @@ -113,17 +113,14 @@ class ResetBudgetJob: counter_key_fn: Callable[[Any], str], log_subject: str, extra_where: Optional[dict] = None, - cache_key_fn: Optional[Callable[[Any], str]] = None, + cache_key_fn: Optional[Callable[[Any], Union[str, List[str]]]] = None, ): """ Generic cascade: zero spend on rows whose budget_id is in the reset set. ``cache_key_fn`` is optional: when provided, after the DB update each - matching row's entry in ``user_api_key_cache`` is also dropped. This - is required for entities whose spend counter is read with the cached - object's ``.spend`` as fallback (tags, end-users) — otherwise the - stale cached object pins enforcement to the pre-reset spend until - its TTL expires. + matching row's entry or entries in ``user_api_key_cache`` are dropped so + cached spend cannot stay pinned above the zeroed DB row after a reset. """ budget_ids = [b.budget_id for b in budgets_to_reset if b.budget_id is not None] if not budget_ids: @@ -146,7 +143,11 @@ class ResetBudgetJob: for row in rows: await self._invalidate_spend_counter(counter_key_fn(row)) if cache_key_fn is not None: - await self._invalidate_user_api_key_cache_entry(cache_key_fn(row)) + cache_keys = cache_key_fn(row) + if isinstance(cache_keys, str): + cache_keys = [cache_keys] + for cache_key in cache_keys: + await self._invalidate_user_api_key_cache_entry(cache_key) return update_result @@ -161,6 +162,7 @@ class ResetBudgetJob: table=self.prisma_client.db.litellm_teammembership, counter_key_fn=lambda m: f"spend:team_member:{m.user_id}:{m.team_id}", log_subject="team memberships", + cache_key_fn=lambda m: f"{m.team_id}_{m.user_id}", ) async def reset_budget_for_keys_linked_to_budgets( @@ -178,6 +180,7 @@ class ResetBudgetJob: counter_key_fn=lambda k: f"spend:key:{k.token}", log_subject="keys", extra_where={"budget_duration": None, "spend": {"gt": 0}}, + cache_key_fn=lambda k: k.token, ) async def reset_budget_for_orgs_linked_to_budgets( @@ -192,6 +195,10 @@ class ResetBudgetJob: counter_key_fn=lambda o: f"spend:org:{o.organization_id}", log_subject="orgs", extra_where={"spend": {"gt": 0}}, + cache_key_fn=lambda o: [ + f"org_id:{o.organization_id}", + f"org_id:{o.organization_id}:with_budget", + ], ) async def reset_budget_for_tags_linked_to_budgets( diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 7bda0f87ccd..f2e64fdde83 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -142,6 +142,7 @@ if MCP_AVAILABLE: MCPOAuthUserCredentialRequest, MCPOAuthUserCredentialStatus, MCPSubmissionsSummary, + MCPTransport, MCPUserCredentialListItem, MCPUserCredentialRequest, MCPUserCredentialResponse, @@ -1070,6 +1071,24 @@ if MCP_AVAILABLE: }, ) + # stdio servers spawn a local subprocess on the proxy host with the + # configured command + args, so accepting them from non-admin callers + # would let a team member propose a server config that an admin could + # rubber-stamp into local code execution. Restrict stdio submission to + # the admin POST /v1/mcp/server path or to config.yaml. + if payload.transport == MCPTransport.stdio: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "error": ( + "stdio MCP servers cannot be submitted via the user " + "registration workflow. Ask a proxy admin to add this " + "server via POST /v1/mcp/server or to declare it in " + "config.yaml." + ) + }, + ) + prisma_client = get_prisma_client_or_throw( "Database not connected. Connect a database to your proxy" ) diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index 6af07585796..409f4fad99a 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -1494,6 +1494,7 @@ async def test_add_update_server_with_alias(): mock_mcp_server.created_at = None mock_mcp_server.updated_at = None mock_mcp_server.instructions = None + mock_mcp_server.approval_status = "active" # Add server to manager await test_manager.add_server(mock_mcp_server) @@ -1551,6 +1552,7 @@ async def test_add_update_server_without_alias(): mock_mcp_server.created_at = None mock_mcp_server.updated_at = None mock_mcp_server.instructions = None + mock_mcp_server.approval_status = "active" # Add server to manager await test_manager.add_server(mock_mcp_server) @@ -1609,6 +1611,7 @@ async def test_add_update_server_fallback_to_server_id(): mock_mcp_server.created_at = None mock_mcp_server.updated_at = None mock_mcp_server.instructions = None + mock_mcp_server.approval_status = "active" # Add server to manager await test_manager.add_server(mock_mcp_server) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 11e9dbbdd57..b53420f0000 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -29,7 +29,11 @@ from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( MCPServerManager, _deserialize_json_dict, ) -from litellm.proxy._types import LiteLLM_MCPServerTable, MCPTransport +from litellm.proxy._types import ( + LiteLLM_MCPServerTable, + MCPApprovalStatus, + MCPTransport, +) from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPOAuthMetadata, MCPServer @@ -3311,5 +3315,126 @@ class TestOAuthDiscoverySSRFGuard: mock_client.get.assert_not_called() +class TestApprovalStatusGate: + """ + Regression tests for GHSA-gm4g-h72v-jhc3. + + The runtime registry must only contain servers an admin has approved. + A non-admin can submit a pending stdio MCP server with an attacker-chosen + command/args; before this gate, an admin opening the per-row endpoint + triggered ``add_server`` + ``health_check_server``, which spawned the + attacker's process under the proxy. The data-layer gate in + ``add_server`` / ``update_server`` blocks pending and rejected rows + from entering the registry regardless of which caller passes them in. + """ + + def _make_server(self, server_id: str, approval_status): + return LiteLLM_MCPServerTable( + server_id=server_id, + alias=f"server_{server_id}", + description="test", + url=None, + transport=MCPTransport.stdio, + command="python", + args=["-c", "print('attacker payload')"], + env={}, + approval_status=approval_status, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + + @pytest.mark.parametrize( + "approval_status,expect_in_registry", + [ + (MCPApprovalStatus.pending_review, False), + (MCPApprovalStatus.rejected, False), + (MCPApprovalStatus.active, True), + # Legacy rows: NULL predates the approval workflow; "approved" is + # a legacy alias for "active" still present in older deployments. + # Both must continue to load to match the DB-level filter in + # reload_servers_from_database(). + (None, True), + ("approved", True), + ], + ) + async def test_add_server_respects_approval_status( + self, approval_status, expect_in_registry + ): + manager = MCPServerManager() + server_id = f"sid-{approval_status}" + await manager.add_server(self._make_server(server_id, approval_status)) + assert (server_id in manager.registry) is expect_in_registry + + async def test_update_server_evicts_when_transitioned_away_from_active(self): + # An admin updates a previously-active server to rejected (or pending). + # The stale registry entry must be evicted so subsequent tool calls + # and health probes can't reach it. + manager = MCPServerManager() + await manager.add_server( + self._make_server("evict-me", MCPApprovalStatus.active) + ) + assert "evict-me" in manager.registry + + await manager.update_server( + self._make_server("evict-me", MCPApprovalStatus.rejected) + ) + assert "evict-me" not in manager.registry + + async def test_update_server_eviction_clears_openapi_routing_artifacts( + self, tmp_path + ): + """Rejecting a server must remove its OpenAPI tools and name mappings.""" + from litellm.proxy._experimental.mcp_server.tool_registry import ( + global_mcp_tool_registry, + ) + from litellm.proxy._experimental.mcp_server.utils import ( + add_server_prefix_to_name, + get_server_prefix, + ) + + manager = MCPServerManager() + await manager.add_server( + self._make_server("evict-openapi", MCPApprovalStatus.active) + ) + assert "evict-openapi" in manager.registry + + server = manager.registry["evict-openapi"] + server.spec_path = str(tmp_path / "unused.yaml") + prefix = get_server_prefix(server) + prefixed = add_server_prefix_to_name("demo_tool", prefix) + + async def _noop_handler(**kwargs): + return None + + global_mcp_tool_registry.register_tool( + name=prefixed, + description="demo", + input_schema={"type": "object"}, + handler=_noop_handler, + ) + manager.tool_name_to_mcp_server_name_mapping["demo_tool"] = prefix + manager.tool_name_to_mcp_server_name_mapping[prefixed] = prefix + + await manager.update_server( + self._make_server("evict-openapi", MCPApprovalStatus.rejected) + ) + + assert "evict-openapi" not in manager.registry + assert prefixed not in global_mcp_tool_registry.tools + assert "demo_tool" not in manager.tool_name_to_mcp_server_name_mapping + assert prefixed not in manager.tool_name_to_mcp_server_name_mapping + + async def test_update_server_noop_for_unregistered_pending(self): + # update_server called with a pending row that was never registered + # should silently return without adding it. Locks in the early-return + # so a future refactor can't accidentally route the pending row to + # build_mcp_server_from_table. + manager = MCPServerManager() + await manager.update_server( + self._make_server("never-seen", MCPApprovalStatus.pending_review) + ) + assert "never-seen" not in manager.registry + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py index 8b0c76f836c..8a47c78db05 100644 --- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py +++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py @@ -1527,15 +1527,21 @@ def test_reset_budget_for_tags_linked_to_budgets_invalidates_each_tag_management assert deleted_keys == {"tag:tenant-a", "tag:tenant-b", "tag:tenant-c"} -def test_reset_budget_for_keys_linked_to_budgets_does_not_touch_management_cache( +def test_reset_budget_for_keys_linked_to_budgets_invalidates_management_cache( monkeypatch, ): - """Cache invalidation is opt-in: keys / orgs / team-members rely on - ``SpendCounterReseed.from_db`` (which DOES handle their counter keys), - so the cache_key_fn hook is intentionally not wired for them. This test - locks in that no-op so a future refactor doesn't accidentally start - clobbering the key cache (which would cost an extra DB round-trip per - reset cycle without fixing anything).""" + """Budget-tier key resets must drop the cached key object (hashed token key). + + Historically this test used ``assert_not_awaited()`` on + ``user_api_key_cache.async_delete_cache``, reflecting the assumption that + ``SpendCounterReseed.from_db`` alone kept spend consistent for keys and + that invalidating the management cache was unnecessary. That was flipped to + ``assert_any_await(...)`` because the old invariant fails across pods: a + budget reset on one instance can leave another pod's cached key object + (including embedded ``.spend``) stale until TTL expiry. Eviction now matches + tags/orgs/teams. Do not treat the ``cache_key_fn`` / invalidation wiring as + redundant without revisiting that cross-pod consistency story. + """ counter_cache = _make_counter_invalidation_job(monkeypatch) expired_budget = type("B", (), {"budget_id": "budget-1"}) @@ -1552,4 +1558,85 @@ def test_reset_budget_for_keys_linked_to_budgets_does_not_touch_management_cache job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) asyncio.run(job.reset_budget_for_keys_linked_to_budgets([expired_budget])) - counter_cache.user_api_key_cache.async_delete_cache.assert_not_awaited() + counter_cache.user_api_key_cache.async_delete_cache.assert_any_await( + key="sk-linked" + ) + + +def test_reset_budget_for_orgs_linked_to_budgets_invalidates_management_cache( + monkeypatch, +): + """Org rows use both base and budget-table cache keys — evict both on reset.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_org = type("Org", (), {"organization_id": "org-acme"}) + + prisma_client = MagicMock() + prisma_client.db.litellm_organizationtable.find_many = AsyncMock( + return_value=[linked_org] + ) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 1} + ) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_orgs_linked_to_budgets([expired_budget])) + + deleted_keys = { + call.kwargs.get("key") + for call in counter_cache.user_api_key_cache.async_delete_cache.await_args_list + } + assert deleted_keys == { + "org_id:org-acme", + "org_id:org-acme:with_budget", + } + + +def test_reset_budget_for_team_members_invalidates_management_cache(monkeypatch): + """Team membership cache key matches auth: ``{team_id}_{user_id}``.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + membership = type( + "Membership", + (), + {"user_id": "alice", "team_id": "team-x", "budget_id": "budget-1"}, + ) + + prisma_client = MagicMock() + prisma_client.db.litellm_teammembership.find_many = AsyncMock( + return_value=[membership] + ) + prisma_client.db.litellm_teammembership.update_many = AsyncMock( + return_value={"count": 1} + ) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_litellm_team_members([expired_budget])) + + counter_cache.user_api_key_cache.async_delete_cache.assert_any_await( + key="team-x_alice" + ) + + +def test_reset_budget_for_tags_linked_to_budgets_management_cache_delete_failure_still_resets( + monkeypatch, +): + """If ``async_delete_cache`` raises, the DB cascade must still complete.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + counter_cache.user_api_key_cache.async_delete_cache = AsyncMock( + side_effect=RuntimeError("cache unavailable") + ) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_tag = type("Tag", (), {"tag_name": "tenant-42"}) + + prisma_client = MagicMock() + prisma_client.db.litellm_tagtable.find_many = AsyncMock(return_value=[linked_tag]) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 1}) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_tags_linked_to_budgets([expired_budget])) + + prisma_client.db.litellm_tagtable.update_many.assert_awaited_once() diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index f0909afcbf6..30ad84e18b8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -2491,6 +2491,32 @@ class TestMCPApprovalWorkflow: assert exc_info.value.status_code == 400 assert "team" in str(exc_info.value.detail).lower() + @pytest.mark.asyncio + async def test_register_mcp_server_rejects_stdio_transport(self): + # stdio servers spawn a local subprocess on the proxy host. Accepting + # them from the non-admin submission endpoint would let a team member + # propose a config that an admin could rubber-stamp into local code + # execution. Admins use POST /v1/mcp/server or config.yaml instead. + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + register_mcp_server, + ) + + payload = NewMCPServerRequest( + alias="local", + transport=MCPTransport.stdio, + command="python3", + args=["-m", "mcp_server_filesystem", "/tmp"], + ) + user_auth = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.INTERNAL_USER, + team_id="team-123", + user_id="user-abc", + ) + with pytest.raises(HTTPException) as exc_info: + await register_mcp_server(payload=payload, user_api_key_dict=user_auth) + assert exc_info.value.status_code == 400 + assert "stdio" in str(exc_info.value.detail).lower() + @pytest.mark.asyncio async def test_register_mcp_server_sets_pending_review(self): from litellm.proxy._types import MCPApprovalStatus From fc8a9a34067bb1571bb02bf6b9dc308f89ba168e Mon Sep 17 00:00:00 2001 From: Jorge Yero Salazar Date: Tue, 12 May 2026 10:25:01 -0500 Subject: [PATCH 82/85] Match litellm.completion supported model parameters with proxy model info (#27720) * Use base_model for supported optional params * Add test * Formatting --- litellm/main.py | 6 ++++- tests/test_litellm/test_main.py | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 52a256fdd05..c3d1c2e05b0 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1528,7 +1528,11 @@ def completion( # type: ignore # noqa: PLR0915 "logit_bias": logit_bias, "user": user, # params to identify the model - "model": model, + "model": ( + model_info.get("base_model") + if isinstance(model_info, dict) and model_info.get("base_model") + else model + ), "custom_llm_provider": custom_llm_provider, "response_format": response_format, "seed": seed, diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 76336a91fc3..b03579c2dbd 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -848,6 +848,45 @@ def test_gpt_5_4_responses_bridge_preserves_reasoning_summary_dict( } +@pytest.mark.parametrize( + "model, model_info, expected_model_param", + [ + ("gemini/gemini-3.1-pro", None, "gemini-3.1-pro"), + ( + "gemini/gemini-3.1-pro", + {"base_model": "gemini-3.1-pro-preview"}, + "gemini-3.1-pro-preview", + ), + ], +) +def test_completion_optional_params_base_model( + model: str, + model_info: dict | None, + expected_model_param: str, +): + with patch("litellm.main.get_optional_params") as mock_get_optional_params: + mock_get_optional_params.return_value = MagicMock() + + import litellm + + kwargs = { + "model": model, + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "api_key": "fake-key", + "mock_response": "Hey, how's it going?", + } + if model_info is not None: + kwargs["model_info"] = model_info + + litellm.completion(**kwargs) + + assert mock_get_optional_params.called is True + get_optional_params_model_param = mock_get_optional_params.call_args.kwargs[ + "model" + ] + assert get_optional_params_model_param == expected_model_param + + @patch("litellm.completion_extras.responses_api_bridge.completion") def test_gpt_5_4_responses_bridge_merges_reasoning_summary_kwarg_without_tools( mock_responses_completion, From b39cac93828e4f1e5c93529de11de6d32c981aab Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 12 May 2026 10:54:55 -0700 Subject: [PATCH 83/85] feat(ui): add Expires to key Overview header; merge User into one field (#27696) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ui): resolve created_by to human-readable name for team keys The team's Virtual Keys table rendered a raw UUID under Created By for keys created on behalf of a real user, and the key details page header showed "-" because it was reading the wrong field (user_email/user_id instead of created_by_user). - TeamVirtualKeysTable Created By column now prefers created_by_user.user_alias > user_email > UUID, with a Popover on hover that surfaces all three fields with copy icons (matches the existing AllKeys table pattern in VirtualKeysTable.tsx) - key_info_view passes the resolved created_by_user value to the details-page header so it renders the readable name Resolves LIT-2517 * fix(ui): add created_by to KeyResponse type The backend returns created_by on every key row, but the frontend type omitted it. The Created By cell already reads the field via info.row.original, which trips the production typecheck. * feat(ui): add Expires to key Overview header; merge User into one field The key details page header omitted Expires (only Settings tab had it) and showed User Email + User ID as two separate rows. This PR: - adds Expires below Created At, reusing the Settings tab's formatTimestamp(...) ?? "Never" formatting so both views agree - merges User Email / User ID into a single "User" field that displays alias / email / user_id (in that fallback order) with a Popover on hover exposing all three with copy icons — mirrors the Created By pattern in TeamVirtualKeysTable.tsx and VirtualKeysTable.tsx Refs LIT-2517 * chore(ui): User field icon + alias-primary test Address review feedback on #27696: - Swap MailOutlined → UserOutlined on the merged User cell; the envelope icon implied "this is an email" but the cell can render alias / email / user_id depending on what's available. - Add a test asserting userAlias displays as primary and overrides userEmail — closes the gap where a fallback-order regression would have silently passed. * fix(ui): truncate long User values and reshuffle header layout - Swap column groupings so Created By stays paired with Created At (matches the pre-merge layout); User and Expires now share col 1. - Ellipsis-truncate the visible User cell at maxWidth 200 so a raw UUID fallback doesn't sprawl. Full identity still revealed via the hover Popover. - Cap each Popover row at maxWidth 220 so a UUID truncates inside the panel too; antd's built-in ellipsis tooltip surfaces the full value. --- .../components/key_team_helpers/key_list.tsx | 1 + .../components/team/TeamVirtualKeysTable.tsx | 56 +++++++-- .../templates/KeyInfoHeader.test.tsx | 35 ++++-- .../components/templates/KeyInfoHeader.tsx | 106 ++++++++++++++++-- .../components/templates/key_info_view.tsx | 8 +- 5 files changed, 179 insertions(+), 27 deletions(-) diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx index f410a049762..6b3c65aaf7b 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx @@ -53,6 +53,7 @@ export interface KeyResponse { organization_id: string | null; org_id?: string | null; created_at: string; + created_by?: string; updated_at: string; last_active: string | null; team_spend: number; diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx index a7e48ff12bc..cfc08e8c4ef 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx @@ -25,7 +25,8 @@ import { Text, } from "@tremor/react"; import { InfoCircleOutlined } from "@ant-design/icons"; -import { Popover, Skeleton, Tooltip } from "antd"; +import { Popover, Skeleton, Tooltip, Typography } from "antd"; +import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; import React, { useCallback, useEffect, useMemo, useState } from "react"; import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; import { KeyResponse, Team } from "../key_team_helpers/key_list"; @@ -339,18 +340,59 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi size: 70, enableSorting: false, cell: (info) => { - const value = info.getValue() as string | null; - const displayValue = value === "default_user_id" ? "Default Proxy Admin" : value; + const userId = info.getValue() as string | null; + if (!userId) return "-"; + const { created_by_user } = info.row.original; + const userAlias = created_by_user?.user_alias ?? null; + const userEmail = created_by_user?.user_email ?? null; + const isDefaultAdmin = userId === "default_user_id"; + const displayValue = userAlias || userEmail || userId; const width = info.cell.column.getSize(); + + const popoverContent = ( +
+ {[ + { label: "User Alias", value: userAlias }, + { label: "User Email", value: userEmail }, + { label: "User ID", value: userId }, + ].map(({ label, value }) => ( +
+ {label} + {value ? ( + + {value} + + ) : ( + - + )} +
+ ))} +
+ ); + + if (isDefaultAdmin && !userAlias && !userEmail) { + return ( + + + + + + ); + } + return ( - + - {displayValue ?? "-"} + {displayValue} - + ); }, }, diff --git a/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.test.tsx b/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.test.tsx index c9c5856129f..14750787609 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.test.tsx @@ -8,10 +8,12 @@ const MOCK_DATA: KeyInfoData = { keyId: "sk-1234567890abcdef", userId: "user-abc-123", userEmail: "test@example.com", + userAlias: null, createdBy: "admin@example.com", createdAt: "Oct 29, 2025 at 1:26 AM", lastUpdated: "Oct 29, 2025 at 1:47 AM", lastActive: "Oct 29, 2025 at 2:00 AM", + expires: "Never", }; describe("KeyInfoHeader", () => { @@ -28,12 +30,11 @@ describe("KeyInfoHeader", () => { it("should render all metadata fields", () => { render(); - expect(screen.getByText("User Email")).toBeInTheDocument(); + expect(screen.getByText("User")).toBeInTheDocument(); expect(screen.getByText("test@example.com")).toBeInTheDocument(); - expect(screen.getByText("User ID")).toBeInTheDocument(); - expect(screen.getByText("user-abc-123")).toBeInTheDocument(); expect(screen.getByText("Created At")).toBeInTheDocument(); expect(screen.getByText("Created By")).toBeInTheDocument(); + expect(screen.getByText("Expires")).toBeInTheDocument(); expect(screen.getByText("Last Updated")).toBeInTheDocument(); expect(screen.getByText("Last Active")).toBeInTheDocument(); }); @@ -122,8 +123,8 @@ describe("KeyInfoHeader", () => { }); describe("default_user_id handling", () => { - it("should show Default Proxy Admin tag for User ID when value is default_user_id", () => { - const data = { ...MOCK_DATA, userId: "default_user_id" }; + it("should show Default Proxy Admin tag for User when userId is default_user_id and no alias/email", () => { + const data = { ...MOCK_DATA, userId: "default_user_id", userEmail: "", userAlias: null }; render(); expect(screen.getAllByText("Default Proxy Admin").length).toBeGreaterThanOrEqual(1); }); @@ -135,9 +136,27 @@ describe("KeyInfoHeader", () => { }); }); - describe("empty value handling", () => { - it("should show '-' for User Email when value is empty", () => { - const data = { ...MOCK_DATA, userEmail: "" }; + describe("User field fallbacks", () => { + it("should display userAlias as primary when set, overriding email and userId", () => { + const data = { ...MOCK_DATA, userAlias: "alice" }; + render(); + expect(screen.getByText("alice")).toBeInTheDocument(); + expect(screen.queryByText("test@example.com")).not.toBeInTheDocument(); + }); + + it("should display userEmail when alias is null", () => { + render(); + expect(screen.getByText("test@example.com")).toBeInTheDocument(); + }); + + it("should fall back to userId when alias and email are missing", () => { + const data = { ...MOCK_DATA, userEmail: "", userAlias: null }; + render(); + expect(screen.getByText("user-abc-123")).toBeInTheDocument(); + }); + + it("should show '-' when alias, email, and userId are all empty", () => { + const data = { ...MOCK_DATA, userId: "", userEmail: "", userAlias: null }; render(); expect(screen.getByText("-")).toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.tsx b/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.tsx index 93ebae9c4be..d39b46a5cdd 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.tsx @@ -1,19 +1,20 @@ import React from "react"; -import { Button, Typography, Tooltip, Space, Divider, Flex } from "antd"; +import { Button, Typography, Tooltip, Space, Divider, Flex, Popover } from "antd"; import { ArrowLeftOutlined, SyncOutlined, DeleteOutlined, PlusOutlined, UserOutlined, - MailOutlined, CalendarOutlined, ClockCircleOutlined, ThunderboltOutlined, SafetyCertificateOutlined, TransactionOutlined, + FieldTimeOutlined, } from "@ant-design/icons"; import LabeledField from "../common_components/LabeledField"; +import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; const { Title, Text } = Typography; @@ -22,10 +23,12 @@ export interface KeyInfoData { keyId: string; userId: string; userEmail: string; + userAlias?: string | null; createdBy: string; createdAt: string; lastUpdated: string; lastActive: string; + expires: string; } interface KeyInfoHeaderProps { @@ -41,6 +44,94 @@ interface KeyInfoHeaderProps { regenerateTooltip?: string; } +function UserField({ + userAlias, + userEmail, + userId, +}: { + userAlias?: string | null; + userEmail: string; + userId: string; +}) { + const labelEl = ( + + + + User + + + ); + + const isEmpty = !userAlias && !userEmail && !userId; + if (isEmpty) { + return ( +
+ {labelEl} +
-
+
+ ); + } + + const isDefaultAdmin = userId === "default_user_id"; + const displayValue = userAlias || userEmail || userId; + + const popoverContent = ( +
+ {[ + { label: "User Alias", value: userAlias ?? null }, + { label: "User Email", value: userEmail || null }, + { label: "User ID", value: userId || null }, + ].map(({ label, value }) => ( +
+ {label} + {value ? ( + + {value} + + ) : ( + - + )} +
+ ))} +
+ ); + + if (isDefaultAdmin && !userAlias && !userEmail) { + return ( +
+ {labelEl} +
+ + + +
+
+ ); + } + + return ( +
+ {labelEl} +
+ + + {displayValue} + + +
+
+ ); +} + export function KeyInfoHeader({ data, onBack, @@ -101,15 +192,8 @@ export function KeyInfoHeader({ - } /> - } - truncate - copyable - defaultUserIdCheck - /> + + } /> diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 65bd9d9eb95..f8897780090 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -403,10 +403,16 @@ export default function KeyInfoView({ keyId: currentKeyData.token_id || currentKeyData.token, userId: currentKeyData.user_id || "", userEmail: currentKeyData.user_email || "", - createdBy: currentKeyData.user_email || currentKeyData.user_id || "", + userAlias: currentKeyData.user?.user_alias ?? null, + createdBy: + currentKeyData.created_by_user?.user_alias || + currentKeyData.created_by_user?.user_email || + currentKeyData.created_by || + "", createdAt: currentKeyData.created_at ? formatTimestamp(currentKeyData.created_at) : "", lastUpdated: currentKeyData.updated_at ? formatTimestamp(currentKeyData.updated_at) : "", lastActive: currentKeyData.last_active ? formatTimestamp(currentKeyData.last_active) : "Never", + expires: currentKeyData.expires ? formatTimestamp(currentKeyData.expires) : "Never", }} onBack={onClose} onRegenerate={() => setIsRegenerateModalOpen(true)} From 9c4faeabc98aea4ac86fd533b823efbb35db79fb Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 12 May 2026 11:14:29 -0700 Subject: [PATCH 84/85] feat(ui): search teams by team ID alongside name (#27684) * feat(ui): search teams by team ID alongside name The Teams page search box only matched team_alias, so pasting a team UUID returned zero results. Detect when the input is a full UUID and route it to the team_id filter instead; otherwise keep the existing alias substring search. Placeholder now reads "Search teams by name or ID...". Resolves LIT-2648 * refactor: search teams via backend OR clause, drop client-side UUID detection Adds a `search` query param to /v2/team/list that ORs across team_id (exact) and team_alias (case-insensitive contains), so the search box sends one param regardless of input format. Removes the isLikelyTeamId helper and the client-side branching it fed. --- .../management_endpoints/team_endpoints.py | 12 ++ .../test_team_endpoints.py | 115 ++++++++++++++++++ .../app/(dashboard)/hooks/teams/useTeams.ts | 3 + .../src/components/OldTeams.tsx | 25 ++-- 4 files changed, 141 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 259624f1e18..65bcca23c30 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3805,6 +3805,7 @@ async def _build_team_list_where_conditions( organization_id: Optional[str], user_id: Optional[str], use_deleted_table: bool, + search: Optional[str] = None, org_admin_org_ids: Optional[List[str]] = None, user_api_key_cache: Optional[Any] = None, proxy_logging_obj: Optional[Any] = None, @@ -3826,6 +3827,12 @@ async def _build_team_list_where_conditions( "mode": "insensitive", # Case-insensitive search } + if search: + where_conditions["OR"] = [ + {"team_id": search}, + {"team_alias": {"contains": search, "mode": "insensitive"}}, + ] + if organization_id: where_conditions["organization_id"] = organization_id elif org_admin_org_ids is not None: @@ -4019,6 +4026,10 @@ async def list_team_v2( default=None, description="Only return teams which this 'team_alias' belongs to. Supports partial matching.", ), + search: Optional[str] = fastapi.Query( + default=None, + description="Combined search: matches teams whose 'team_id' equals the value OR whose 'team_alias' contains it (case-insensitive).", + ), page: int = fastapi.Query( default=1, description="Page number for pagination", ge=1 ), @@ -4104,6 +4115,7 @@ async def list_team_v2( organization_id=organization_id, user_id=user_id, use_deleted_table=use_deleted_table, + search=search, org_admin_org_ids=org_admin_org_ids, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index e668672dd2a..5c7bbc46c95 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -3140,6 +3140,121 @@ async def test_list_team_v2_with_invalid_status(): assert "deleted" in str(exc_info.value.detail) +@pytest.mark.asyncio +async def test_list_team_v2_search_builds_or_clause(): + """ + `search` should be passed as a Prisma OR across team_id (exact) and + team_alias (case-insensitive contains), so the UI can hit a single + backend filter with either a UUID or a name fragment. + """ + from unittest.mock import AsyncMock, Mock, patch + + from fastapi import Request + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import list_team_v2 + + mock_request = Mock(spec=Request) + mock_admin = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma_client: + mock_db = Mock() + mock_prisma_client.db = mock_db + mock_db.litellm_teamtable.find_many = AsyncMock(return_value=[]) + mock_db.litellm_teamtable.count = AsyncMock(return_value=0) + + await list_team_v2( + http_request=mock_request, + user_id=None, + organization_id=None, + team_id=None, + team_alias=None, + search="platform", + user_api_key_dict=mock_admin, + page=1, + page_size=10, + status=None, + ) + + find_many_kwargs = mock_db.litellm_teamtable.find_many.call_args.kwargs + assert find_many_kwargs["where"] == { + "OR": [ + {"team_id": "platform"}, + {"team_alias": {"contains": "platform", "mode": "insensitive"}}, + ] + } + + +@pytest.mark.asyncio +async def test_list_team_v2_search_composes_with_user_id_filter(): + """ + For non-admin users, `search` must compose with the membership filter: + the resulting where clause should AND `team_id IN ` with + the search OR clause, so users still only see their own teams. + """ + from datetime import datetime + from unittest.mock import AsyncMock, Mock, patch + + from fastapi import Request + + from litellm.proxy._types import ( + LiteLLM_UserTable, + LitellmUserRoles, + UserAPIKeyAuth, + ) + from litellm.proxy.management_endpoints.team_endpoints import list_team_v2 + + mock_request = Mock(spec=Request) + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="member_user" + ) + + mock_user = LiteLLM_UserTable( + user_id="member_user", + teams=["team_a", "team_b"], + organization_memberships=[], + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.management_endpoints.team_endpoints.get_user_object", + new=AsyncMock(return_value=mock_user), + ), + patch( + "litellm.proxy.management_endpoints.team_endpoints._get_org_admin_org_ids", + new=AsyncMock(return_value=None), + ), + ): + mock_db = Mock() + mock_prisma.db = mock_db + mock_db.litellm_teamtable.find_many = AsyncMock(return_value=[]) + mock_db.litellm_teamtable.count = AsyncMock(return_value=0) + + await list_team_v2( + http_request=mock_request, + user_id="member_user", + organization_id=None, + team_id=None, + team_alias=None, + search="team_a", + user_api_key_dict=mock_user_api_key_dict, + page=1, + page_size=10, + status=None, + ) + + find_many_kwargs = mock_db.litellm_teamtable.find_many.call_args.kwargs + where = find_many_kwargs["where"] + assert where["OR"] == [ + {"team_id": "team_a"}, + {"team_alias": {"contains": "team_a", "mode": "insensitive"}}, + ] + assert where["team_id"] == {"in": ["team_a", "team_b"]} + + @pytest.mark.asyncio async def test_team_member_delete_cleans_membership(mock_db_client, mock_admin_auth): """ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts index f74a71e901e..b25b6ce393a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts @@ -29,6 +29,7 @@ export interface TeamListCallOptions { organizationID?: string | null; teamID?: string | null; team_alias?: string | null; + search?: string | null; userID?: string | null; sortBy?: string | null; sortOrder?: string | null; @@ -52,6 +53,7 @@ export const teamListCall = async ( team_id: options.teamID, organization_id: options.organizationID, team_alias: options.team_alias, + search: options.search, user_id: options.userID, page, page_size: pageSize, @@ -178,6 +180,7 @@ const deletedTeamListCall = async ( team_id: options.teamID, organization_id: options.organizationID, team_alias: options.team_alias, + search: options.search, user_id: options.userID, page, page_size: pageSize, diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index abc26c4cd44..b9305e4723a 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -80,8 +80,7 @@ interface TeamProps { } interface FilterState { - team_id: string; - team_alias: string; + search: string; organization_id: string; sort_by: string; sort_order: "asc" | "desc"; @@ -200,8 +199,7 @@ const Teams: React.FC = ({ const [currentOrg, setCurrentOrg] = useState(null); const [currentOrgForCreateTeam, setCurrentOrgForCreateTeam] = useState(null); const [filters, setFilters] = useState({ - team_id: "", - team_alias: "", + search: "", organization_id: "", sort_by: "created_at", sort_order: "desc", @@ -215,7 +213,7 @@ const Teams: React.FC = ({ sortBy?: string; sortOrder?: string; organizationID?: string; - teamAlias?: string; + search?: string; } = {}) => { if (!accessToken) return; const page = opts.page ?? currentPage; @@ -223,7 +221,7 @@ const Teams: React.FC = ({ const sortBy = opts.sortBy ?? filters.sort_by; const sortOrder = opts.sortOrder ?? filters.sort_order; const organizationID = opts.organizationID ?? filters.organization_id; - const teamAlias = opts.teamAlias ?? filters.team_alias; + const search = opts.search ?? filters.search; setIsLoading(true); setFetchError(null); @@ -234,7 +232,7 @@ const Teams: React.FC = ({ size, { organizationID: organizationID || null, - team_alias: teamAlias || null, + search: search || null, userID: userRole !== "Admin" && userRole !== "Admin Viewer" ? userID : null, sortBy: sortBy || null, sortOrder: sortOrder || null, @@ -632,9 +630,9 @@ const Teams: React.FC = ({ setIsSearching(true); searchDebounceRef.current = setTimeout(async () => { try { - setFilters((prev) => ({ ...prev, team_alias: value })); + setFilters((prev) => ({ ...prev, search: value })); setCurrentPage(1); - await fetchTeamsV2({ page: 1, teamAlias: value }); + await fetchTeamsV2({ page: 1, search: value }); } finally { setIsSearching(false); } @@ -653,7 +651,7 @@ const Teams: React.FC = ({ pageSize, { organizationID: newFilters.organization_id || null, - team_alias: newFilters.team_alias || null, + search: newFilters.search || null, userID: userRole !== "Admin" && userRole !== "Admin Viewer" ? userID : null, sortBy: newFilters.sort_by || null, sortOrder: newFilters.sort_order || null, @@ -670,15 +668,14 @@ const Teams: React.FC = ({ if (searchDebounceRef.current) clearTimeout(searchDebounceRef.current); setIsSearching(false); const resetFilters: FilterState = { - team_id: "", - team_alias: "", + search: "", organization_id: "", sort_by: "created_at", sort_order: "desc", }; setFilters(resetFilters); setCurrentPage(1); - fetchTeamsV2({ page: 1, organizationID: "", teamAlias: "", sortBy: "created_at", sortOrder: "desc" }); + fetchTeamsV2({ page: 1, organizationID: "", search: "", sortBy: "created_at", sortOrder: "desc" }); }; const { token } = theme.useToken(); @@ -945,7 +942,7 @@ const Teams: React.FC = ({ } suffix={isSearching ? : null} - placeholder="Search teams by name..." + placeholder="Search teams by name or ID..." onChange={(e) => handleSearchChange(e.target.value)} allowClear style={{ maxWidth: 400 }} From 63a2d1ddc94ce382566e90ac65c0367346cb1db9 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 12 May 2026 12:32:57 -0700 Subject: [PATCH 85/85] fix(tests): use canonical litellm_enterprise import path (#27699) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The enterprise package is installed as `litellm_enterprise` (per enterprise/pyproject.toml), but several tests imported it as `enterprise.litellm_enterprise.*` — a path that only resolves because the repo root happens to sit on sys.path, letting Python's implicit namespace package machinery discover `enterprise/` as a directory. This breaks any test runner that relocates source (e.g. the mutation-testing workflow, which copies tests under `mutants/`) and also caused two `patch()` strings to target a module path that does not match what production code imports — meaning those mocks were never actually patching the production module's attribute. Replace `from enterprise.litellm_enterprise.` with the canonical `from litellm_enterprise.` across 6 test files, and fix two `patch()` target strings (and one `sys.modules` patch key in the SSO test) to match. --- .../test_internal_user_endpoints.py | 65 ++-- .../send_emails/test_base_email.py | 181 ++++++---- .../test_callback_controls.py | 334 +++++++++++++----- .../llms/test_file_search_responses.py | 6 +- .../guardrails/test_guardrail_coverage.py | 4 +- .../test_project_org_authz.py | 6 +- .../proxy/management_endpoints/test_ui_sso.py | 12 +- 7 files changed, 406 insertions(+), 202 deletions(-) diff --git a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_internal_user_endpoints.py index 69c3b4cb59a..32685c5cbd3 100644 --- a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_internal_user_endpoints.py @@ -19,7 +19,7 @@ def client(): def mock_user_api_key_auth(): """Mock the user_api_key_auth dependency""" with patch( - "enterprise.litellm_enterprise.proxy.management_endpoints.internal_user_endpoints.user_api_key_auth" + "litellm_enterprise.proxy.management_endpoints.internal_user_endpoints.user_api_key_auth" ) as mock_auth: mock_auth.return_value = {"user_id": "test_user", "api_key": "test_key"} yield mock_auth @@ -31,12 +31,16 @@ class TestAvailableEnterpriseUsers: self, client, mock_user_api_key_auth ): """Test when max_users is set and user count is within limit""" - with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, patch( - "litellm.proxy.proxy_server.premium_user", - True, - ), patch( - "litellm.proxy.proxy_server.premium_user_data", - {"max_users": 10}, + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.proxy_server.premium_user", + True, + ), + patch( + "litellm.proxy.proxy_server.premium_user_data", + {"max_users": 10}, + ), ): # Mock database count mock_prisma.db.litellm_usertable.count = AsyncMock(return_value=5) @@ -66,12 +70,16 @@ class TestAvailableEnterpriseUsers: self, client, mock_user_api_key_auth ): """Test when max_users is not set (premium_user_data is None or doesn't contain max_users)""" - with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, patch( - "litellm.proxy.proxy_server.premium_user", - True, - ), patch( - "litellm.proxy.proxy_server.premium_user_data", - None, + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.proxy_server.premium_user", + True, + ), + patch( + "litellm.proxy.proxy_server.premium_user_data", + None, + ), ): # Mock database count mock_prisma.db.litellm_usertable.count = AsyncMock(return_value=3) @@ -99,12 +107,16 @@ class TestAvailableEnterpriseUsers: self, client, mock_user_api_key_auth ): """Test the current bug where total_users_remaining can be negative""" - with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, patch( - "litellm.proxy.proxy_server.premium_user", - True, - ), patch( - "litellm.proxy.proxy_server.premium_user_data", - {"key": "value"}, + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.proxy_server.premium_user", + True, + ), + patch( + "litellm.proxy.proxy_server.premium_user_data", + {"key": "value"}, + ), ): # Mock database count higher than max_users to trigger the bug mock_prisma.db.litellm_usertable.count = AsyncMock(return_value=8) @@ -140,12 +152,15 @@ class TestAvailableEnterpriseUsers: """Test when prisma_client is None (no database connection)""" from litellm.proxy._types import CommonProxyErrors - with patch( - "litellm.proxy.proxy_server.prisma_client", - None, - ), patch( - "litellm.proxy.proxy_server.premium_user", - True, + with ( + patch( + "litellm.proxy.proxy_server.prisma_client", + None, + ), + patch( + "litellm.proxy.proxy_server.premium_user", + True, + ), ): # Override the dependency client.app.dependency_overrides[mock_user_api_key_auth] = lambda: { diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py index af5e2341406..5cabfe5fb7f 100644 --- a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py +++ b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py @@ -7,7 +7,7 @@ from unittest.mock import patch import pytest from fastapi.testclient import TestClient -from enterprise.litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( +from litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( BaseEmailLogger, ) @@ -30,6 +30,7 @@ def no_invitation_wait(monkeypatch): monkeypatch.setattr(BaseEmailLogger, "_wait_for_invitation_creation", _noop) + @pytest.fixture def base_email_logger(): return BaseEmailLogger() @@ -283,7 +284,10 @@ async def test_send_key_created_email_without_key( mock_send_email.assert_called_once() call_args = mock_send_email.call_args[1] assert "sk-secret-key-456" not in call_args["html_body"] - assert "[Key hidden for security - retrieve from dashboard]" in call_args["html_body"] + assert ( + "[Key hidden for security - retrieve from dashboard]" + in call_args["html_body"] + ) @pytest.mark.asyncio @@ -317,7 +321,10 @@ async def test_send_key_rotated_email_without_key( mock_send_email.assert_called_once() call_args = mock_send_email.call_args[1] assert "sk-secret-rotated-789" not in call_args["html_body"] - assert "[Key hidden for security - retrieve from dashboard]" in call_args["html_body"] + assert ( + "[Key hidden for security - retrieve from dashboard]" + in call_args["html_body"] + ) @pytest.mark.asyncio @@ -371,52 +378,52 @@ async def test_get_invitation_link_creates_new_when_none_exist(base_email_logger """Test that _get_invitation_link creates a new invitation when none exist""" # Mock prisma client with no existing invitation rows mock_prisma = mock.MagicMock() - + # Mock find_many to return empty list (no existing invitations) async def mock_find_many_empty(*args, **kwargs): return [] - + mock_prisma.db.litellm_invitationlink.find_many = mock_find_many_empty - + # Mock the create_invitation_for_user function mock_created_invitation = mock.MagicMock() mock_created_invitation.id = "new-invitation-id" - + with mock.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): with mock.patch( "litellm.proxy.management_helpers.user_invitation.create_invitation_for_user", - return_value=mock_created_invitation + return_value=mock_created_invitation, ) as mock_create_invitation: # Execute result = await base_email_logger._get_invitation_link( user_id="test-user", base_url="http://test.com" ) - + # Verify that create_invitation_for_user was called mock_create_invitation.assert_called_once() call_args = mock_create_invitation.call_args[1] assert call_args["data"].user_id == "test-user" assert call_args["user_api_key_dict"].user_id == "test-user" - + # Verify the returned link uses the new invitation ID assert result == "http://test.com/ui?invitation_id=new-invitation-id" -@pytest.mark.asyncio +@pytest.mark.asyncio async def test_get_invitation_link_uses_existing_when_available(base_email_logger): """Test that _get_invitation_link uses existing invitation when available""" # Mock prisma client with existing invitation row mock_invitation_row = mock.MagicMock() mock_invitation_row.id = "existing-invitation-id" - + mock_prisma = mock.MagicMock() - + # Mock find_many to return existing invitation async def mock_find_many_existing(*args, **kwargs): return [mock_invitation_row] - + mock_prisma.db.litellm_invitationlink.find_many = mock_find_many_existing - + with mock.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): with mock.patch( "litellm.proxy.management_helpers.user_invitation.create_invitation_for_user" @@ -425,10 +432,10 @@ async def test_get_invitation_link_uses_existing_when_available(base_email_logge result = await base_email_logger._get_invitation_link( user_id="test-user", base_url="http://test.com" ) - + # Verify that create_invitation_for_user was NOT called mock_create_invitation.assert_not_called() - + # Verify the returned link uses the existing invitation ID assert result == "http://test.com/ui?invitation_id=existing-invitation-id" @@ -438,33 +445,33 @@ async def test_get_invitation_link_creates_new_when_list_is_none(base_email_logg """Test that _get_invitation_link creates a new invitation when invitation_rows is None""" # Mock prisma client to return None mock_prisma = mock.MagicMock() - + # Mock find_many to return None async def mock_find_many_none(*args, **kwargs): return None - + mock_prisma.db.litellm_invitationlink.find_many = mock_find_many_none - + # Mock the create_invitation_for_user function mock_created_invitation = mock.MagicMock() mock_created_invitation.id = "new-invitation-from-none" - + with mock.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): with mock.patch( "litellm.proxy.management_helpers.user_invitation.create_invitation_for_user", - return_value=mock_created_invitation + return_value=mock_created_invitation, ) as mock_create_invitation: # Execute result = await base_email_logger._get_invitation_link( user_id="test-user", base_url="http://test.com" ) - + # Verify that create_invitation_for_user was called mock_create_invitation.assert_called_once() call_args = mock_create_invitation.call_args[1] assert call_args["data"].user_id == "test-user" assert call_args["user_api_key_dict"].user_id == "test-user" - + # Verify the returned link uses the new invitation ID assert result == "http://test.com/ui?invitation_id=new-invitation-from-none" @@ -495,13 +502,15 @@ async def test_get_email_params_user_invitation( user_email="test@example.com", ) - assert result.logo_url == "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" + assert ( + result.logo_url + == "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" + ) assert result.support_contact == "support@berri.ai" assert result.base_url == "http://test.com/ui?invitation_id=test-id" assert result.recipient_email == "test@example.com" - @pytest.fixture def mock_env_vars(monkeypatch): """Set up test environment variables""" @@ -513,69 +522,74 @@ def mock_env_vars(monkeypatch): monkeypatch.setenv("PROXY_BASE_URL", "http://test.com") monkeypatch.setenv("PROXY_API_URL", "https://test.com") + @pytest.mark.asyncio async def test_get_email_params_custom_templates_premium_user(mock_env_vars): """Test that _get_email_params returns correct values with custom templates for premium users""" # Mock premium_user as True with patch("litellm.proxy.proxy_server.premium_user", True): email_logger = BaseEmailLogger() - + # Test invitation email params invitation_params = await email_logger._get_email_params( email_event=EmailEvent.new_user_invitation, user_id="testid", user_email="test@example.com", - event_message="New User Invitation" + event_message="New User Invitation", ) - + assert invitation_params.subject == "Welcome to Test Company!" assert invitation_params.signature == "Best regards,\nTest Company Team" assert invitation_params.logo_url == "https://test-company.com/logo.png" assert invitation_params.support_contact == "support@test-company.com" assert invitation_params.base_url == "http://test.com" - + # Test key created email params key_params = await email_logger._get_email_params( email_event=EmailEvent.virtual_key_created, user_id="testid", user_email="test@example.com", - event_message="API Key Created" + event_message="API Key Created", ) - + assert key_params.subject == "Your Test Company API Key" assert key_params.signature == "Best regards,\nTest Company Team" + @pytest.mark.asyncio async def test_get_email_params_non_premium_user(mock_env_vars): """Test that non-premium users get default templates even when custom ones are provided""" # Mock premium_user as False with patch("litellm.proxy.proxy_server.premium_user", False): email_logger = BaseEmailLogger() - + # Test invitation email params email_params = await email_logger._get_email_params( email_event=EmailEvent.new_user_invitation, user_email="test@example.com", - event_message="New User Invitation" + event_message="New User Invitation", ) - + # Should use default values even though custom values are set in env assert email_params.subject == "LiteLLM: New User Invitation" assert email_params.signature == EMAIL_FOOTER - assert email_params.logo_url == "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" + assert ( + email_params.logo_url + == "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" + ) assert email_params.support_contact == "support@berri.ai" - # Test key created email params key_params = await email_logger._get_email_params( email_event=EmailEvent.virtual_key_created, user_email="test@example.com", - event_message="API Key Created" + event_message="API Key Created", ) - + assert key_params.subject == "LiteLLM: API Key Created" assert key_params.signature == EMAIL_FOOTER + @pytest.mark.asyncio async def test_get_email_params_default_templates(monkeypatch): """Test that _get_email_params uses default templates when custom ones aren't provided""" @@ -583,28 +597,28 @@ async def test_get_email_params_default_templates(monkeypatch): monkeypatch.delenv("EMAIL_SUBJECT_INVITATION", raising=False) monkeypatch.delenv("EMAIL_SUBJECT_KEY_CREATED", raising=False) monkeypatch.delenv("EMAIL_SIGNATURE", raising=False) - + # Mock premium_user as True (shouldn't matter since no custom values are set) with patch("litellm.proxy.proxy_server.premium_user", True): email_logger = BaseEmailLogger() - + # Test invitation email params with default template invitation_params = await email_logger._get_email_params( email_event=EmailEvent.new_user_invitation, user_email="test@example.com", - event_message="New User Invitation" + event_message="New User Invitation", ) - + assert invitation_params.subject == "LiteLLM: New User Invitation" assert invitation_params.signature == EMAIL_FOOTER - + # Test key created email params with default template key_params = await email_logger._get_email_params( email_event=EmailEvent.virtual_key_created, user_email="test@example.com", - event_message="API Key Created" + event_message="API Key Created", ) - + assert key_params.subject == "LiteLLM: API Key Created" assert key_params.signature == EMAIL_FOOTER @@ -639,7 +653,10 @@ async def test_send_soft_budget_alert_email( call_args = mock_send_email.call_args[1] assert call_args["from_email"] == BaseEmailLogger.DEFAULT_LITELLM_EMAIL assert call_args["to_email"] == ["test@example.com"] - assert call_args["subject"] == "LiteLLM: Soft Budget Crossed - Total Soft Budget: $100.0" + assert ( + call_args["subject"] + == "LiteLLM: Soft Budget Crossed - Total Soft Budget: $100.0" + ) assert "$100.0" in call_args["html_body"] # soft_budget assert "$105.0" in call_args["html_body"] # spend assert "$200.0" in call_args["html_body"] # max_budget @@ -673,13 +690,13 @@ async def test_send_soft_budget_alert_email_no_max_budget( call_args = mock_send_email.call_args[1] assert "$100.0" in call_args["html_body"] # soft_budget assert "$105.0" in call_args["html_body"] # spend - assert "Maximum Budget" not in call_args["html_body"] # max_budget should not be shown + assert ( + "Maximum Budget" not in call_args["html_body"] + ) # max_budget should not be shown @pytest.mark.asyncio -async def test_budget_alerts_soft_budget_crossed( - base_email_logger, mock_send_email -): +async def test_budget_alerts_soft_budget_crossed(base_email_logger, mock_send_email): """Test that budget_alerts sends email when soft budget is crossed""" user_info = CallInfo( user_id="test_user", @@ -708,11 +725,14 @@ async def test_budget_alerts_soft_budget_crossed( mock_send_email.assert_called_once() call_args = mock_send_email.call_args[1] assert call_args["to_email"] == ["test@example.com"] - + # Verify cache was set to prevent duplicate alerts mock_cache.async_set_cache.assert_called_once() cache_call_args = mock_cache.async_set_cache.call_args[1] - assert cache_call_args["key"] == "email_budget_alerts:soft_budget_crossed:test_user" + assert ( + cache_call_args["key"] + == "email_budget_alerts:soft_budget_crossed:test_user" + ) assert cache_call_args["value"] == "SENT" assert cache_call_args["ttl"] == EMAIL_BUDGET_ALERT_TTL @@ -766,9 +786,7 @@ async def test_budget_alerts_soft_budget_duplicate_prevention( @pytest.mark.asyncio -async def test_budget_alerts_no_budgets( - base_email_logger, mock_send_email -): +async def test_budget_alerts_no_budgets(base_email_logger, mock_send_email): """Test that budget_alerts returns early when no budgets are set""" user_info = CallInfo( user_id="test_user", @@ -817,7 +835,10 @@ async def test_budget_alerts_uses_token_for_cache_key( # Verify cache key uses token instead of user_id mock_cache.async_set_cache.assert_called_once() cache_call_args = mock_cache.async_set_cache.call_args[1] - assert cache_call_args["key"] == "email_budget_alerts:soft_budget_crossed:hashed_token_123" + assert ( + cache_call_args["key"] + == "email_budget_alerts:soft_budget_crossed:hashed_token_123" + ) @pytest.mark.asyncio @@ -838,7 +859,9 @@ async def test_get_email_params_soft_budget_crossed( ) # Should use default subject template for soft_budget_crossed - assert result.subject == "LiteLLM: Soft Budget Crossed - Total Soft Budget: $100.0" + assert ( + result.subject == "LiteLLM: Soft Budget Crossed - Total Soft Budget: $100.0" + ) assert result.recipient_email == "test@example.com" assert result.base_url == "http://test.com" @@ -867,16 +890,20 @@ async def test_budget_alerts_max_budget_alert_crossed( "PROXY_BASE_URL": "http://test.com", }, ): - await base_email_logger.budget_alerts(type="max_budget_alert", user_info=user_info) + await base_email_logger.budget_alerts( + type="max_budget_alert", user_info=user_info + ) mock_send_email.assert_called_once() call_args = mock_send_email.call_args[1] assert call_args["to_email"] == ["test@example.com"] assert "Max Budget Alert" in call_args["subject"] - + mock_cache.async_set_cache.assert_called_once() cache_call_args = mock_cache.async_set_cache.call_args[1] - assert cache_call_args["key"] == "email_budget_alerts:max_budget_alert:test_user" + assert ( + cache_call_args["key"] == "email_budget_alerts:max_budget_alert:test_user" + ) assert cache_call_args["value"] == "SENT" assert cache_call_args["ttl"] == EMAIL_BUDGET_ALERT_TTL @@ -906,15 +933,15 @@ async def test_multi_threshold_sends_crossed_thresholds( base_email_logger.internal_usage_cache = mock_cache with mock.patch.dict(os.environ, {"PROXY_BASE_URL": "http://test.com"}): - await base_email_logger.budget_alerts(type="max_budget_alert", user_info=user_info) + await base_email_logger.budget_alerts( + type="max_budget_alert", user_info=user_info + ) # spend=80 crosses 50% ($50) and 75% ($75), but not 100% ($100) assert mock_send_email.call_count == 2 # Check cache keys include threshold percentage - cache_keys = [ - c[1]["key"] for c in mock_cache.async_set_cache.call_args_list - ] + cache_keys = [c[1]["key"] for c in mock_cache.async_set_cache.call_args_list] assert "email_budget_alerts:max_budget_alert:50:hashed_key_1" in cache_keys assert "email_budget_alerts:max_budget_alert:75:hashed_key_1" in cache_keys @@ -949,7 +976,9 @@ async def test_multi_threshold_dedup_cache_prevents_resend( base_email_logger.internal_usage_cache = mock_cache with mock.patch.dict(os.environ, {"PROXY_BASE_URL": "http://test.com"}): - await base_email_logger.budget_alerts(type="max_budget_alert", user_info=user_info) + await base_email_logger.budget_alerts( + type="max_budget_alert", user_info=user_info + ) # Only 75% should fire assert mock_send_email.call_count == 1 @@ -980,7 +1009,9 @@ async def test_multi_threshold_owner_email_auto_included( base_email_logger.internal_usage_cache = mock_cache with mock.patch.dict(os.environ, {"PROXY_BASE_URL": "http://test.com"}): - await base_email_logger.budget_alerts(type="max_budget_alert", user_info=user_info) + await base_email_logger.budget_alerts( + type="max_budget_alert", user_info=user_info + ) mock_send_email.assert_called_once() to_emails = mock_send_email.call_args[1]["to_email"] @@ -1002,7 +1033,7 @@ async def test_multi_threshold_malformed_keys_skipped( event_group=Litellm_EntityType.KEY, max_budget_alert_emails={ "fifty": ["finance@co.com"], # invalid - "50": ["finance@co.com"], # valid, crossed + "50": ["finance@co.com"], # valid, crossed }, ) @@ -1012,7 +1043,9 @@ async def test_multi_threshold_malformed_keys_skipped( base_email_logger.internal_usage_cache = mock_cache with mock.patch.dict(os.environ, {"PROXY_BASE_URL": "http://test.com"}): - await base_email_logger.budget_alerts(type="max_budget_alert", user_info=user_info) + await base_email_logger.budget_alerts( + type="max_budget_alert", user_info=user_info + ) # Only the valid "50" threshold should fire assert mock_send_email.call_count == 1 @@ -1041,7 +1074,9 @@ async def test_multi_threshold_empty_emails_only_owner( base_email_logger.internal_usage_cache = mock_cache with mock.patch.dict(os.environ, {"PROXY_BASE_URL": "http://test.com"}): - await base_email_logger.budget_alerts(type="max_budget_alert", user_info=user_info) + await base_email_logger.budget_alerts( + type="max_budget_alert", user_info=user_info + ) mock_send_email.assert_called_once() to_emails = mock_send_email.call_args[1]["to_email"] @@ -1067,11 +1102,13 @@ async def test_no_map_preserves_old_single_threshold( base_email_logger.internal_usage_cache = mock_cache with mock.patch.dict(os.environ, {"PROXY_BASE_URL": "http://test.com"}): - await base_email_logger.budget_alerts(type="max_budget_alert", user_info=user_info) + await base_email_logger.budget_alerts( + type="max_budget_alert", user_info=user_info + ) mock_send_email.assert_called_once() call_args = mock_send_email.call_args[1] assert call_args["to_email"] == ["test@example.com"] # Old path cache key has no threshold percentage cache_key = mock_cache.async_set_cache.call_args[1]["key"] - assert cache_key == "email_budget_alerts:max_budget_alert:test_user" \ No newline at end of file + assert cache_key == "email_budget_alerts:max_budget_alert:test_user" diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/test_callback_controls.py b/tests/test_litellm/enterprise/enterprise_callbacks/test_callback_controls.py index b160ca5130c..d67dc3cf6bc 100644 --- a/tests/test_litellm/enterprise/enterprise_callbacks/test_callback_controls.py +++ b/tests/test_litellm/enterprise/enterprise_callbacks/test_callback_controls.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch import pytest -from enterprise.litellm_enterprise.enterprise_callbacks.callback_controls import ( +from litellm_enterprise.enterprise_callbacks.callback_controls import ( EnterpriseCallbackControls, ) from litellm.constants import X_LITELLM_DISABLE_CALLBACKS @@ -18,168 +18,282 @@ from litellm.types.utils import StandardCallbackDynamicParams class TestEnterpriseCallbackControls: - + @pytest.fixture def mock_premium_user(self): """Fixture to mock premium user check as True""" - with patch.object(EnterpriseCallbackControls, '_should_allow_dynamic_callback_disabling', return_value=True): + with patch.object( + EnterpriseCallbackControls, + "_should_allow_dynamic_callback_disabling", + return_value=True, + ): yield - - @pytest.fixture + + @pytest.fixture def mock_non_premium_user(self): """Fixture to mock premium user check as False""" - with patch.object(EnterpriseCallbackControls, '_should_allow_dynamic_callback_disabling', return_value=False): + with patch.object( + EnterpriseCallbackControls, + "_should_allow_dynamic_callback_disabling", + return_value=False, + ): yield @pytest.fixture def mock_request_headers(self): """Fixture to mock get_proxy_server_request_headers""" - with patch('enterprise.litellm_enterprise.enterprise_callbacks.callback_controls.get_proxy_server_request_headers') as mock_headers: + with patch( + "litellm_enterprise.enterprise_callbacks.callback_controls.get_proxy_server_request_headers" + ) as mock_headers: yield mock_headers - def test_callback_disabled_langfuse_string(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_langfuse_string( + self, mock_premium_user, mock_request_headers + ): """Test that 'langfuse' string callback is disabled when specified in headers""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is True - def test_callback_disabled_langfuse_customlogger(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_langfuse_customlogger( + self, mock_premium_user, mock_request_headers + ): """Test that LangfusePromptManagement CustomLogger instance is disabled when 'langfuse' specified in headers""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + langfuse_logger = LangfusePromptManagement() - result = EnterpriseCallbackControls.is_callback_disabled_dynamically(langfuse_logger, litellm_params, standard_callback_dynamic_params) + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + langfuse_logger, litellm_params, standard_callback_dynamic_params + ) assert result is True - def test_callback_disabled_s3_v2_string(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_s3_v2_string( + self, mock_premium_user, mock_request_headers + ): """Test that 's3_v2' string callback is disabled when specified in headers""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "s3_v2"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("s3_v2", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "s3_v2", litellm_params, standard_callback_dynamic_params + ) assert result is True - def test_callback_disabled_s3_v2_customlogger(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_s3_v2_customlogger( + self, mock_premium_user, mock_request_headers + ): """Test that S3Logger CustomLogger instance is disabled when 's3_v2' specified in headers""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "s3_v2"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + # Mock S3Logger to avoid async initialization issues - with patch('litellm.integrations.s3_v2.S3Logger.__init__', return_value=None): + with patch("litellm.integrations.s3_v2.S3Logger.__init__", return_value=None): s3_logger = S3Logger() - result = EnterpriseCallbackControls.is_callback_disabled_dynamically(s3_logger, litellm_params, standard_callback_dynamic_params) + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + s3_logger, litellm_params, standard_callback_dynamic_params + ) assert result is True - def test_callback_disabled_datadog_string(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_datadog_string( + self, mock_premium_user, mock_request_headers + ): """Test that 'datadog' string callback is disabled when specified in headers""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "datadog"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("datadog", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "datadog", litellm_params, standard_callback_dynamic_params + ) assert result is True - def test_callback_disabled_datadog_customlogger(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_datadog_customlogger( + self, mock_premium_user, mock_request_headers + ): """Test that DataDogLogger CustomLogger instance is disabled when 'datadog' specified in headers""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "datadog"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + # Mock DataDogLogger to avoid async initialization issues - with patch('litellm.integrations.datadog.datadog.DataDogLogger.__init__', return_value=None): + with patch( + "litellm.integrations.datadog.datadog.DataDogLogger.__init__", + return_value=None, + ): datadog_logger = DataDogLogger() - result = EnterpriseCallbackControls.is_callback_disabled_dynamically(datadog_logger, litellm_params, standard_callback_dynamic_params) + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + datadog_logger, litellm_params, standard_callback_dynamic_params + ) assert result is True def test_multiple_callbacks_disabled(self, mock_premium_user, mock_request_headers): """Test that multiple callbacks can be disabled with comma-separated list""" - mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse,datadog,s3_v2"} + mock_request_headers.return_value = { + X_LITELLM_DISABLE_CALLBACKS: "langfuse,datadog,s3_v2" + } litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - # Test each callback is disabled - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) is True - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("datadog", litellm_params, standard_callback_dynamic_params) is True - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("s3_v2", litellm_params, standard_callback_dynamic_params) is True - - # Test non-disabled callback is not disabled - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("prometheus", litellm_params, standard_callback_dynamic_params) is False - def test_callback_not_disabled_when_not_in_list(self, mock_premium_user, mock_request_headers): + # Test each callback is disabled + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) + is True + ) + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "datadog", litellm_params, standard_callback_dynamic_params + ) + is True + ) + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "s3_v2", litellm_params, standard_callback_dynamic_params + ) + is True + ) + + # Test non-disabled callback is not disabled + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "prometheus", litellm_params, standard_callback_dynamic_params + ) + is False + ) + + def test_callback_not_disabled_when_not_in_list( + self, mock_premium_user, mock_request_headers + ): """Test that callbacks not in the disabled list are not disabled""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("datadog", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "datadog", litellm_params, standard_callback_dynamic_params + ) assert result is False - def test_callback_not_disabled_when_no_header(self, mock_premium_user, mock_request_headers): + def test_callback_not_disabled_when_no_header( + self, mock_premium_user, mock_request_headers + ): """Test that callbacks are not disabled when the header is not present""" mock_request_headers.return_value = {} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is False - def test_callback_not_disabled_when_header_none(self, mock_premium_user, mock_request_headers): + def test_callback_not_disabled_when_header_none( + self, mock_premium_user, mock_request_headers + ): """Test that callbacks are not disabled when the header value is None""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: None} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is False - def test_non_premium_user_cannot_disable_callbacks(self, mock_non_premium_user, mock_request_headers): + def test_non_premium_user_cannot_disable_callbacks( + self, mock_non_premium_user, mock_request_headers + ): """Test that non-premium users cannot disable callbacks even with the header""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is False - def test_case_insensitive_callback_matching(self, mock_premium_user, mock_request_headers): + def test_case_insensitive_callback_matching( + self, mock_premium_user, mock_request_headers + ): """Test that callback matching is case insensitive""" - mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "LANGFUSE,DataDog"} + mock_request_headers.return_value = { + X_LITELLM_DISABLE_CALLBACKS: "LANGFUSE,DataDog" + } litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + # Test lowercase callbacks are disabled - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) is True - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("datadog", litellm_params, standard_callback_dynamic_params) is True + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) + is True + ) + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "datadog", litellm_params, standard_callback_dynamic_params + ) + is True + ) - def test_whitespace_handling_in_disabled_callbacks(self, mock_premium_user, mock_request_headers): + def test_whitespace_handling_in_disabled_callbacks( + self, mock_premium_user, mock_request_headers + ): """Test that whitespace around callback names is handled correctly""" - mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: " langfuse , datadog , s3_v2 "} + mock_request_headers.return_value = { + X_LITELLM_DISABLE_CALLBACKS: " langfuse , datadog , s3_v2 " + } litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) is True - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("datadog", litellm_params, standard_callback_dynamic_params) is True - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("s3_v2", litellm_params, standard_callback_dynamic_params) is True - def test_custom_logger_not_in_registry(self, mock_premium_user, mock_request_headers): + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) + is True + ) + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "datadog", litellm_params, standard_callback_dynamic_params + ) + is True + ) + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "s3_v2", litellm_params, standard_callback_dynamic_params + ) + is True + ) + + def test_custom_logger_not_in_registry( + self, mock_premium_user, mock_request_headers + ): """Test that CustomLogger not in registry is not disabled""" - mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "unknown_logger"} + mock_request_headers.return_value = { + X_LITELLM_DISABLE_CALLBACKS: "unknown_logger" + } litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + # Create a mock CustomLogger that's not in the registry class UnknownLogger(CustomLogger): pass - + unknown_logger = UnknownLogger() - result = EnterpriseCallbackControls.is_callback_disabled_dynamically(unknown_logger, litellm_params, standard_callback_dynamic_params) + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + unknown_logger, litellm_params, standard_callback_dynamic_params + ) assert result is False def test_exception_handling(self, mock_premium_user, mock_request_headers): @@ -188,32 +302,64 @@ class TestEnterpriseCallbackControls: mock_request_headers.side_effect = Exception("Test exception") litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is False - def test_callback_disabled_via_request_body_langfuse(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_via_request_body_langfuse( + self, mock_premium_user, mock_request_headers + ): """Test that callbacks can be disabled via request body litellm_disabled_callbacks""" mock_request_headers.return_value = {} # No headers litellm_params = {"proxy_server_request": {"url": "test"}} - standard_callback_dynamic_params = StandardCallbackDynamicParams(litellm_disabled_callbacks=["langfuse"]) - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + standard_callback_dynamic_params = StandardCallbackDynamicParams( + litellm_disabled_callbacks=["langfuse"] + ) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is True - def test_callback_disabled_via_request_body_multiple(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_via_request_body_multiple( + self, mock_premium_user, mock_request_headers + ): """Test that multiple callbacks can be disabled via request body""" mock_request_headers.return_value = {} # No headers litellm_params = {"proxy_server_request": {"url": "test"}} - standard_callback_dynamic_params = StandardCallbackDynamicParams(litellm_disabled_callbacks=["langfuse", "datadog", "s3_v2"]) - + standard_callback_dynamic_params = StandardCallbackDynamicParams( + litellm_disabled_callbacks=["langfuse", "datadog", "s3_v2"] + ) + # Test each callback is disabled - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) is True - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("datadog", litellm_params, standard_callback_dynamic_params) is True - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("s3_v2", litellm_params, standard_callback_dynamic_params) is True - + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) + is True + ) + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "datadog", litellm_params, standard_callback_dynamic_params + ) + is True + ) + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "s3_v2", litellm_params, standard_callback_dynamic_params + ) + is True + ) + # Test non-disabled callback is not disabled - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("prometheus", litellm_params, standard_callback_dynamic_params) is False + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "prometheus", litellm_params, standard_callback_dynamic_params + ) + is False + ) def test_admin_can_disable_dynamic_callback_disabling(self, mock_request_headers): """ @@ -223,11 +369,13 @@ class TestEnterpriseCallbackControls: mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + # Mock litellm.allow_dynamic_callback_disabling set to False - with patch('litellm.allow_dynamic_callback_disabling', False): - with patch('litellm.proxy.proxy_server.premium_user', True): - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + with patch("litellm.allow_dynamic_callback_disabling", False): + with patch("litellm.proxy.proxy_server.premium_user", True): + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is False def test_admin_can_enable_dynamic_callback_disabling(self, mock_request_headers): @@ -238,14 +386,18 @@ class TestEnterpriseCallbackControls: mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + # Mock litellm.allow_dynamic_callback_disabling set to True - with patch('litellm.allow_dynamic_callback_disabling', True): - with patch('litellm.proxy.proxy_server.premium_user', True): - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + with patch("litellm.allow_dynamic_callback_disabling", True): + with patch("litellm.proxy.proxy_server.premium_user", True): + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is True - def test_default_admin_setting_allows_dynamic_callback_disabling(self, mock_request_headers): + def test_default_admin_setting_allows_dynamic_callback_disabling( + self, mock_request_headers + ): """ Test that when allow_dynamic_callback_disabling is not set, it defaults to True and allows dynamic callback disabling for premium users @@ -253,8 +405,10 @@ class TestEnterpriseCallbackControls: mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + # litellm.allow_dynamic_callback_disabling should default to True - with patch('litellm.proxy.proxy_server.premium_user', True): - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + with patch("litellm.proxy.proxy_server.premium_user", True): + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is True diff --git a/tests/test_litellm/llms/test_file_search_responses.py b/tests/test_litellm/llms/test_file_search_responses.py index 9943b456083..2f7ad3874fa 100644 --- a/tests/test_litellm/llms/test_file_search_responses.py +++ b/tests/test_litellm/llms/test_file_search_responses.py @@ -327,7 +327,7 @@ class TestFileSearchGuardInResponsesMain: class TestManagedFilesVectorStoreAccess: def _make_hook(self): """Return a ManagedFiles instance with prisma_client mocked.""" - from enterprise.litellm_enterprise.proxy.hooks.managed_files import ( + from litellm_enterprise.proxy.hooks.managed_files import ( _PROXY_LiteLLMManagedFiles as ManagedFiles, ) @@ -471,7 +471,7 @@ class TestManagedFilesVectorStoreAccess: @pytest.mark.asyncio async def test_F6_non_responses_call_type_skipped(self): """Access check only runs for aresponses/responses call types.""" - from enterprise.litellm_enterprise.proxy.hooks.managed_files import ( + from litellm_enterprise.proxy.hooks.managed_files import ( _PROXY_LiteLLMManagedFiles as ManagedFiles, ) from litellm.proxy._types import CallTypes @@ -499,7 +499,7 @@ class TestManagedFilesVectorStoreAccess: class TestGetVectorStoreIdsFromFileSearchTools: def _make_hook(self): - from enterprise.litellm_enterprise.proxy.hooks.managed_files import ( + from litellm_enterprise.proxy.hooks.managed_files import ( _PROXY_LiteLLMManagedFiles as ManagedFiles, ) diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_coverage.py b/tests/test_litellm/proxy/guardrails/test_guardrail_coverage.py index 6def548b93f..4c19ee2906b 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_coverage.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_coverage.py @@ -660,7 +660,7 @@ async def test_azure_content_safety_post_call_checks_all_choices(user_api_key): @pytest.mark.asyncio async def test_secret_detection_redacts_multimodal_text_parts(user_api_key): - from enterprise.litellm_enterprise.enterprise_callbacks.secret_detection import ( + from litellm_enterprise.enterprise_callbacks.secret_detection import ( _ENTERPRISE_SecretDetection, ) @@ -696,7 +696,7 @@ async def test_secret_detection_redacts_multimodal_text_parts(user_api_key): @pytest.mark.asyncio async def test_secret_detection_redacts_responses_api_input(user_api_key): - from enterprise.litellm_enterprise.enterprise_callbacks.secret_detection import ( + from litellm_enterprise.enterprise_callbacks.secret_detection import ( _ENTERPRISE_SecretDetection, ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_project_org_authz.py b/tests/test_litellm/proxy/management_endpoints/test_project_org_authz.py index bd982480d60..a06d79306ab 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_project_org_authz.py +++ b/tests/test_litellm/proxy/management_endpoints/test_project_org_authz.py @@ -34,7 +34,7 @@ async def test_project_perm_check_uses_current_team_not_caller_supplied(): """The permission check must look at the project's existing team. Even if the caller is admin of an unrelated team, they must not pass when no explicit team_object is forced through.""" - from enterprise.litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( _check_user_permission_for_project, ) @@ -56,7 +56,7 @@ async def test_project_perm_check_uses_current_team_not_caller_supplied(): @pytest.mark.asyncio async def test_project_perm_check_allows_team_admin_of_existing_team(): - from enterprise.litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( _check_user_permission_for_project, ) @@ -76,7 +76,7 @@ async def test_project_perm_check_allows_team_admin_of_existing_team(): @pytest.mark.asyncio async def test_project_perm_check_proxy_admin_always_allowed(): - from enterprise.litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( _check_user_permission_for_project, ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 2dcf6becbb1..83317157847 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -1797,15 +1797,13 @@ class TestCustomUISSO: ): with patch.dict( "sys.modules", - { - "enterprise.litellm_enterprise.proxy.auth.custom_sso_handler": None - }, + {"litellm_enterprise.proxy.auth.custom_sso_handler": None}, ): # Temporarily mock the google_login function call to test the import error path async def mock_google_login(): # This mimics the relevant part of google_login that would trigger the import error try: - from enterprise.litellm_enterprise.proxy.auth.custom_sso_handler import ( # noqa: F401 + from litellm_enterprise.proxy.auth.custom_sso_handler import ( # noqa: F401 EnterpriseCustomSSOHandler, ) @@ -1828,7 +1826,7 @@ class TestCustomUISSO: """Test successful custom UI SSO sign-in with valid headers""" from fastapi_sso.sso.base import OpenID - from enterprise.litellm_enterprise.proxy.auth.custom_sso_handler import ( + from litellm_enterprise.proxy.auth.custom_sso_handler import ( EnterpriseCustomSSOHandler, ) from litellm.integrations.custom_sso_handler import CustomSSOLoginHandler @@ -1903,7 +1901,7 @@ class TestCustomUISSO: @pytest.mark.asyncio async def test_handle_custom_ui_sso_sign_in_rejects_untrusted_proxy(self): """Custom UI SSO rejects spoofed identity headers from direct clients.""" - from enterprise.litellm_enterprise.proxy.auth.custom_sso_handler import ( + from litellm_enterprise.proxy.auth.custom_sso_handler import ( EnterpriseCustomSSOHandler, ) from litellm.integrations.custom_sso_handler import CustomSSOLoginHandler @@ -1943,7 +1941,7 @@ class TestCustomUISSO: """ from fastapi_sso.sso.base import OpenID - from enterprise.litellm_enterprise.proxy.auth.custom_sso_handler import ( + from litellm_enterprise.proxy.auth.custom_sso_handler import ( EnterpriseCustomSSOHandler, ) from litellm.integrations.custom_sso_handler import CustomSSOLoginHandler