From 5d7b7e7e373ab116a94aba655af28253a2b4282a Mon Sep 17 00:00:00 2001 From: Michael Riad Zaky Date: Wed, 6 May 2026 12:32:30 -0700 Subject: [PATCH 01/23] fix(realtime): register /openai/v1/realtime as websocket route --- litellm/proxy/_types.py | 2 ++ litellm/proxy/proxy_server.py | 1 + litellm/types/utils.py | 1 + tests/test_litellm/proxy/test_proxy_server.py | 34 +++++++++++++++++++ 4 files changed, 38 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c6653a722d6..d777d34eaff 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -353,8 +353,10 @@ class LiteLLMRoutes(enum.Enum): # realtime "/realtime", "/v1/realtime", + "/openai/v1/realtime", "/realtime?{model}", "/v1/realtime?{model}", + "/openai/v1/realtime?{model}", # responses API "/responses", "/v1/responses", diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a5905765c6e..3f7745a54c0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -8805,6 +8805,7 @@ def _realtime_query_params_template( return tuple(params) +@app.websocket("/openai/v1/realtime") @app.websocket("/v1/realtime") @app.websocket("/realtime") async def realtime_websocket_endpoint( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 00a7748309b..400edcac889 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -825,6 +825,7 @@ API_ROUTE_TO_CALL_TYPES = { # Realtime API "/realtime": [CallTypes.arealtime], "/v1/realtime": [CallTypes.arealtime], + "/openai/v1/realtime": [CallTypes.arealtime], # Provider-specific routes "/anthropic/v1/messages": [CallTypes.anthropic_messages], # Google GenAI routes diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 3f19db36c3f..6718f52cbf1 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -6513,3 +6513,37 @@ async def test_get_current_spend_redis_error_falls_back_to_in_memory(): finally: ps.spend_counter_cache = orig_counter ps.prisma_client = orig_prisma + + +def test_realtime_websocket_route_aliases_registered(): + """Realtime sessions reach the proxy via three path aliases stacked on + `realtime_websocket_endpoint`. Dropping any of them silently 405s + WebSocket upgrades because the catch-all `/openai/{endpoint:path}` + HTTP passthrough only declares HTTP methods. The aliases must also be + in `LiteLLMRoutes.openai_routes` (so non-admin / team / key-scoped + auth allows them) and in `API_ROUTE_TO_CALL_TYPES` (so call-type-aware + logic such as guardrails can resolve the realtime call type).""" + from starlette.routing import WebSocketRoute + + from litellm.proxy._types import LiteLLMRoutes + from litellm.proxy.proxy_server import app + from litellm.types.utils import API_ROUTE_TO_CALL_TYPES, CallTypes + + websocket_paths = { + route.path for route in app.routes if isinstance(route, WebSocketRoute) + } + openai_routes = LiteLLMRoutes.openai_routes.value + + for expected in ("/openai/v1/realtime", "/v1/realtime", "/realtime"): + assert expected in websocket_paths, ( + f"{expected!r} missing from registered WebSocket routes; the " + f"realtime endpoint will 405 for clients hitting this path." + ) + assert expected in openai_routes, ( + f"{expected!r} missing from LiteLLMRoutes.openai_routes; " + f"non-admin / team / key-scoped users will get 403 on this path." + ) + assert API_ROUTE_TO_CALL_TYPES.get(expected) == [CallTypes.arealtime], ( + f"{expected!r} missing from API_ROUTE_TO_CALL_TYPES; call-type " + f"resolution will return None and break call-type-aware features." + ) From 32b031aa2cc13c9e06c6ac9fc40940c0c9514e97 Mon Sep 17 00:00:00 2001 From: Milan Date: Thu, 7 May 2026 01:20:03 +0300 Subject: [PATCH 02/23] Flush virtual-key model max budget increments to Redis after success logging. _PROXY_VirtualKeyModelMaxBudgetLimiter subclasses RouterBudgetLimiting but does not run its __init__, so the periodic Redis sync task never starts and spend stayed in memory. Push the increment pipeline when Redis is configured so multi-worker enforcement and cache keys stay consistent. Co-authored-by: Cursor --- litellm/proxy/hooks/model_max_budget_limiter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index 95ffafb7bad..9286424878c 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -319,6 +319,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): response_cost=response_cost, ) + if self.dual_cache.redis_cache is not None: + await self._push_in_memory_increments_to_redis() + verbose_proxy_logger.debug( "current state of in memory cache %s", json.dumps( From fa4c7a2ac6948cbcc975611231cea1608bf04b3b Mon Sep 17 00:00:00 2001 From: Milan Date: Thu, 7 May 2026 01:21:19 +0300 Subject: [PATCH 03/23] Add unit tests for virtual-key model max budget Redis flush. Assert _push_in_memory_increments_to_redis runs after async_log_success_event when dual_cache.redis_cache is set, and is skipped when Redis is not configured. Co-authored-by: Cursor --- ...test_unit_test_max_model_budget_limiter.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py index b4aac113f57..0daa5b17ffa 100644 --- a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py +++ b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py @@ -413,3 +413,71 @@ async def test_async_log_success_event_uses_end_user_model_budget_duration( f"{END_USER_SPEND_CACHE_KEY_PREFIX}:{end_user_id}:{model}:{budget_duration}" ) assert call_kwargs["response_cost"] == 0.05 + + +@pytest.mark.asyncio +async def test_async_log_success_event_pushes_redis_increments_when_redis_configured(): + """ + Virtual-key model max budget limiter does not run RouterBudgetLimiting.__init__, + so the periodic Redis flush task never starts. After logging spend we must call + _push_in_memory_increments_to_redis when Redis is wired so other workers see spend. + """ + dual_cache = DualCache() + dual_cache.redis_cache = object() # truthy placeholder; push only checks is not None + limiter = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=dual_cache) + model = "gpt-4" + kwargs = { + "standard_logging_object": { + "response_cost": 0.01, + "model": model, + "metadata": {"user_api_key_hash": "vk-hash"}, + }, + "litellm_params": { + "metadata": { + "user_api_key_model_max_budget": { + model: {"budget_limit": 10.0, "time_period": "1d"}, + }, + }, + }, + } + with patch.object(limiter, "_increment_spend_for_key", new_callable=AsyncMock): + with patch.object( + limiter, + "_push_in_memory_increments_to_redis", + new_callable=AsyncMock, + ) as mock_push: + await limiter.async_log_success_event( + kwargs, response_obj=None, start_time=None, end_time=None + ) + mock_push.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_log_success_event_skips_redis_push_without_redis(budget_limiter): + """When dual_cache has no Redis backend, do not await _push_in_memory_increments_to_redis.""" + assert budget_limiter.dual_cache.redis_cache is None + model = "gpt-4" + kwargs = { + "standard_logging_object": { + "response_cost": 0.01, + "model": model, + "metadata": {"user_api_key_hash": "vk-hash"}, + }, + "litellm_params": { + "metadata": { + "user_api_key_model_max_budget": { + model: {"budget_limit": 10.0, "time_period": "1d"}, + }, + }, + }, + } + with patch.object(budget_limiter, "_increment_spend_for_key", new_callable=AsyncMock): + with patch.object( + budget_limiter, + "_push_in_memory_increments_to_redis", + new_callable=AsyncMock, + ) as mock_push: + await budget_limiter.async_log_success_event( + kwargs, response_obj=None, start_time=None, end_time=None + ) + mock_push.assert_not_awaited() From 581ae1443d3e80af03030641fe7674e8e12d6c65 Mon Sep 17 00:00:00 2001 From: Michael Riad Zaky Date: Thu, 7 May 2026 09:44:39 -0700 Subject: [PATCH 04/23] honor OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT --- litellm/integrations/opentelemetry.py | 91 ++++++++++++-- .../integrations/test_opentelemetry.py | 111 ++++++++++++++++++ 2 files changed, 193 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 77833e5de0f..e49f0fabdaa 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -57,6 +57,17 @@ LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request" RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request" LITELLM_REQUEST_SPAN_NAME = "litellm_request" +CAPTURE_MODE_NO_CONTENT = "NO_CONTENT" +CAPTURE_MODE_SPAN_ONLY = "SPAN_ONLY" +CAPTURE_MODE_EVENT_ONLY = "EVENT_ONLY" +CAPTURE_MODE_SPAN_AND_EVENT = "SPAN_AND_EVENT" +_VALID_CAPTURE_MODES = { + CAPTURE_MODE_NO_CONTENT, + CAPTURE_MODE_SPAN_ONLY, + CAPTURE_MODE_EVENT_ONLY, + CAPTURE_MODE_SPAN_AND_EVENT, +} + @dataclass class OpenTelemetryConfig: @@ -71,6 +82,9 @@ class OpenTelemetryConfig: ignore_context_propagation: Optional[bool] = None # When True, create a private TracerProvider instead of reusing or setting the global one. skip_set_global: bool = False + # Programmatic override for OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT. + # One of NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT (or "true" as legacy alias). + capture_message_content: Optional[str] = None def __post_init__(self) -> None: # If endpoint is specified but exporter is still the default "console", @@ -182,6 +196,9 @@ class OpenTelemetry(CustomLogger): super().__init__(**kwargs) self._init_metrics(meter_provider) self._init_logs(logger_provider) + # Sample env-var / config / message_logging at init so subsequent + # _capture_in_span / _capture_in_event calls are deterministic. + self._capture_mode_cached = self._compute_capture_mode_from_init_state() self._init_otel_logger_on_litellm_proxy() @staticmethod @@ -306,6 +323,59 @@ class OpenTelemetry(CustomLogger): hasattr(self, "callback_name") and self.callback_name == "langfuse_otel" ) + def _compute_capture_mode_from_init_state(self) -> Optional[str]: + """Sample explicit settings at init. Returns the resolved mode or + None if nothing explicit is set (in which case the legacy + ``self.message_logging`` flag is consulted dynamically per request). + + ``"true"``/``"1"`` map to ``EVENT_ONLY`` per the contrib convention. + Unknown values are ignored. + """ + explicit = self.config.capture_message_content or os.getenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT" + ) + if not explicit: + return None + normalized = explicit.upper() + if normalized in ("TRUE", "1"): + return CAPTURE_MODE_EVENT_ONLY + if normalized in _VALID_CAPTURE_MODES: + return normalized + return None + + def _resolve_capture_mode(self) -> str: + """Return the active capture mode for this request. + + Precedence: + 1. ``litellm.turn_off_message_logging=True`` forces ``NO_CONTENT`` + (kill-switch checked dynamically). + 2. Explicit setting sampled at init from + ``OpenTelemetryConfig.capture_message_content`` or + ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT``. + 3. Legacy ``self.message_logging`` (checked dynamically). + """ + if litellm.turn_off_message_logging: + return CAPTURE_MODE_NO_CONTENT + if self._capture_mode_cached is not None: + return self._capture_mode_cached + return ( + CAPTURE_MODE_SPAN_AND_EVENT + if self.message_logging + else CAPTURE_MODE_NO_CONTENT + ) + + def _capture_in_span(self) -> bool: + return self._resolve_capture_mode() in ( + CAPTURE_MODE_SPAN_ONLY, + CAPTURE_MODE_SPAN_AND_EVENT, + ) + + def _capture_in_event(self) -> bool: + return self._resolve_capture_mode() in ( + CAPTURE_MODE_EVENT_ONLY, + CAPTURE_MODE_SPAN_AND_EVENT, + ) + def _init_tracing(self, tracer_provider): from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider @@ -825,8 +895,7 @@ class OpenTelemetry(CustomLogger): from opentelemetry import trace from opentelemetry.trace import Status, StatusCode - # only log raw LLM request/response if message_logging is on and not globally turned off - if litellm.turn_off_message_logging or not self.message_logging: + if not self._capture_in_span(): return litellm_params = kwargs.get("litellm_params", {}) @@ -1117,9 +1186,14 @@ class OpenTelemetry(CustomLogger): } if role == "tool" and msg.get("id"): attrs["id"] = msg["id"] - if self.message_logging and msg.get("content"): + capture_event_content = self._capture_in_event() + if capture_event_content and msg.get("content"): attrs["gen_ai.prompt"] = msg["content"] + body = msg.copy() + if not capture_event_content: + body.pop("content", None) + log_record = SdkLogRecord( timestamp=self._to_ns(datetime.now()), trace_id=parent_ctx.trace_id, @@ -1127,7 +1201,7 @@ class OpenTelemetry(CustomLogger): trace_flags=parent_ctx.trace_flags, severity_number=SeverityNumber.INFO, severity_text="INFO", - body=msg.copy(), + body=body, attributes=attrs, ) otel_logger.emit(log_record) @@ -1141,14 +1215,15 @@ class OpenTelemetry(CustomLogger): "finish_reason": choice.get("finish_reason"), } body_msg = choice.get("message", {}) - if self.message_logging and body_msg.get("content"): + capture_event_content = self._capture_in_event() + if capture_event_content and body_msg.get("content"): attrs["message.content"] = body_msg["content"] body = { "index": idx, "finish_reason": choice.get("finish_reason"), "message": {"role": body_msg.get("role", "assistant")}, } - if self.message_logging and body_msg.get("content"): + if capture_event_content and body_msg.get("content"): body["message"]["content"] = body_msg["content"] log_record = SdkLogRecord( @@ -1674,9 +1749,7 @@ class OpenTelemetry(CustomLogger): ########## LLM Request Medssages / tools / content Attributes ########### ######################################################################### - if litellm.turn_off_message_logging is True: - return - if self.message_logging is not True: + if not self._capture_in_span(): return if optional_params.get("tools"): diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 962806c5b52..97e57240a96 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -442,6 +442,103 @@ class TestOpenTelemetryDualHandlerIsolation(unittest.TestCase): ) +class TestOpenTelemetryCaptureMessageContent(unittest.TestCase): + """OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT and the + OpenTelemetryConfig.capture_message_content programmatic override + drive what the handler captures in spans vs events.""" + + @staticmethod + def _make(env=None, config_value=None, message_logging=True): + env_dict = ( + {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": env} + if env is not None + else {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": ""} + ) + with patch.dict(os.environ, env_dict): + handler = OpenTelemetry( + config=OpenTelemetryConfig( + exporter="console", capture_message_content=config_value + ) + ) + handler.message_logging = message_logging + return handler, handler._resolve_capture_mode() + + def test_no_explicit_setting_falls_back_to_message_logging_true(self): + _, mode = self._make() + self.assertEqual(mode, "SPAN_AND_EVENT") + + def test_no_explicit_setting_falls_back_to_message_logging_false(self): + _, mode = self._make(message_logging=False) + self.assertEqual(mode, "NO_CONTENT") + + def test_env_var_no_content(self): + _, mode = self._make(env="NO_CONTENT") + self.assertEqual(mode, "NO_CONTENT") + + def test_env_var_span_only(self): + _, mode = self._make(env="SPAN_ONLY") + self.assertEqual(mode, "SPAN_ONLY") + + def test_env_var_event_only(self): + _, mode = self._make(env="EVENT_ONLY") + self.assertEqual(mode, "EVENT_ONLY") + + def test_env_var_span_and_event(self): + _, mode = self._make(env="SPAN_AND_EVENT") + self.assertEqual(mode, "SPAN_AND_EVENT") + + def test_env_var_legacy_true_maps_to_event_only(self): + _, mode = self._make(env="true") + self.assertEqual(mode, "EVENT_ONLY") + + def test_env_var_unknown_value_falls_through_to_legacy(self): + _, mode = self._make(env="garbage", message_logging=True) + self.assertEqual(mode, "SPAN_AND_EVENT") + + def test_config_field_overrides_env(self): + _, mode = self._make(env="EVENT_ONLY", config_value="SPAN_ONLY") + self.assertEqual(mode, "SPAN_ONLY") + + def test_turn_off_message_logging_forces_no_content(self): + with patch("litellm.turn_off_message_logging", True): + _, mode = self._make(env="SPAN_AND_EVENT", message_logging=True) + self.assertEqual(mode, "NO_CONTENT") + + def test_capture_in_span_and_event_predicates(self): + cases = { + "NO_CONTENT": (False, False), + "SPAN_ONLY": (True, False), + "EVENT_ONLY": (False, True), + "SPAN_AND_EVENT": (True, True), + } + for mode, (in_span, in_event) in cases.items(): + handler, _ = self._make(env=mode) + self.assertEqual(handler._capture_in_span(), in_span, msg=mode) + self.assertEqual(handler._capture_in_event(), in_event, msg=mode) + + def test_two_handlers_can_have_different_modes(self): + # FIL's stated requirement: one handler strips content, the other keeps it. + with patch.dict( + os.environ, {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": ""} + ): + stripped = OpenTelemetry( + config=OpenTelemetryConfig( + exporter="console", capture_message_content="NO_CONTENT" + ) + ) + kept = OpenTelemetry( + config=OpenTelemetryConfig( + exporter="console", capture_message_content="SPAN_AND_EVENT" + ) + ) + self.assertEqual(stripped._resolve_capture_mode(), "NO_CONTENT") + self.assertEqual(kept._resolve_capture_mode(), "SPAN_AND_EVENT") + self.assertFalse(stripped._capture_in_span()) + self.assertFalse(stripped._capture_in_event()) + self.assertTrue(kept._capture_in_span()) + self.assertTrue(kept._capture_in_event()) + + class TestOpenTelemetry(unittest.TestCase): POLL_INTERVAL = 0.05 POLL_TIMEOUT = 2.0 @@ -1067,6 +1164,7 @@ class TestOpenTelemetry(unittest.TestCase): result = otel._get_span_name(kwargs) self.assertEqual(result, LITELLM_REQUEST_SPAN_NAME) + @patch.dict(os.environ, {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": ""}) @patch("litellm.turn_off_message_logging", False) def test_maybe_log_raw_request_creates_span(self): """Test _maybe_log_raw_request creates span when logging enabled""" @@ -2194,6 +2292,19 @@ class TestOpenTelemetrySemanticConventions138(unittest.TestCase): See: https://github.com/BerriAI/litellm/issues/17794 """ + def setUp(self): + # Insulate from a shell-set OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT + # so these tests exercise the legacy default path (message_logging=True). + self._prev = os.environ.pop( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", None + ) + + def tearDown(self): + if self._prev is not None: + os.environ["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"] = ( + self._prev + ) + def test_input_messages_uses_parts_structure(self): """ Test that gen_ai.input.messages uses the OTEL 1.38 parts array structure. From 492118de2515a2e2b327013c7004080e3d36d87d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 7 May 2026 18:12:23 +0000 Subject: [PATCH 05/23] Fix OTEL false content capture mode --- litellm/integrations/opentelemetry.py | 3 +++ tests/test_litellm/integrations/test_opentelemetry.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index e49f0fabdaa..0af016feb26 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -329,6 +329,7 @@ class OpenTelemetry(CustomLogger): ``self.message_logging`` flag is consulted dynamically per request). ``"true"``/``"1"`` map to ``EVENT_ONLY`` per the contrib convention. + ``"false"``/``"0"`` map to ``NO_CONTENT``. Unknown values are ignored. """ explicit = self.config.capture_message_content or os.getenv( @@ -339,6 +340,8 @@ class OpenTelemetry(CustomLogger): normalized = explicit.upper() if normalized in ("TRUE", "1"): return CAPTURE_MODE_EVENT_ONLY + if normalized in ("FALSE", "0"): + return CAPTURE_MODE_NO_CONTENT if normalized in _VALID_CAPTURE_MODES: return normalized return None diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 97e57240a96..bea718dfea0 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -491,6 +491,12 @@ class TestOpenTelemetryCaptureMessageContent(unittest.TestCase): _, mode = self._make(env="true") self.assertEqual(mode, "EVENT_ONLY") + def test_env_var_legacy_false_maps_to_no_content(self): + for env in ("false", "0"): + with self.subTest(env=env): + _, mode = self._make(env=env) + self.assertEqual(mode, "NO_CONTENT") + def test_env_var_unknown_value_falls_through_to_legacy(self): _, mode = self._make(env="garbage", message_logging=True) self.assertEqual(mode, "SPAN_AND_EVENT") From f58f8927f27ced15d9e27dcb46cfef530d670f52 Mon Sep 17 00:00:00 2001 From: shivam Date: Thu, 7 May 2026 18:39:26 -0700 Subject: [PATCH 06/23] feat(guardrails): optional skip tool message in unified guardrail inputs Mirrors the system-message skip in PR #25481 for tool-role messages. Adds a global litellm.skip_tool_message_in_guardrail flag and a per-guardrail litellm_params.skip_tool_message_in_guardrail override, applied in the OpenAI and Anthropic chat translation handlers. Co-Authored-By: Claude Opus 4.7 (1M context) --- litellm/__init__.py | 1 + .../chat/guardrail_translation/handler.py | 12 +- .../base_llm/guardrail_translation/utils.py | 15 ++ .../chat/guardrail_translation/handler.py | 24 +++- .../proxy/guardrails/guardrail_registry.py | 5 + litellm/types/guardrails.py | 10 ++ .../test_unified_guardrail.py | 132 ++++++++++++++++++ 7 files changed, 192 insertions(+), 7 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index cf05fc4c980..fd3d47ec154 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -206,6 +206,7 @@ add_user_information_to_llm_headers: Optional[bool] = ( ) store_audit_logs = False # Enterprise feature, allow users to see audit logs skip_system_message_in_guardrail: bool = False +skip_tool_message_in_guardrail: bool = False ### end of callbacks ############# email: Optional[str] = ( diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 2bb82f227bb..74dadee5ecb 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -23,7 +23,9 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, openai_messages_without_system, + openai_messages_without_tool, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, @@ -108,6 +110,7 @@ class AnthropicMessagesHandler(BaseTranslation): return data skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply) + skip_tool = effective_skip_tool_message_for_guardrail(guardrail_to_apply) chat_completion_compatible_request = self._translate_to_openai(data) @@ -117,6 +120,8 @@ class AnthropicMessagesHandler(BaseTranslation): ) if skip_system: structured_messages = openai_messages_without_system(structured_messages) + if skip_tool: + structured_messages = openai_messages_without_tool(structured_messages) texts_to_check: List[str] = [] images_to_check: List[str] = [] @@ -134,6 +139,7 @@ class AnthropicMessagesHandler(BaseTranslation): images_to_check=images_to_check, task_mappings=task_mappings, skip_system_message=skip_system, + skip_tool_message=skip_tool, ) # Step 2: Apply guardrail to all texts in batch @@ -198,13 +204,17 @@ class AnthropicMessagesHandler(BaseTranslation): images_to_check: List[str], task_mappings: List[Tuple[int, Optional[int]]], skip_system_message: bool = False, + skip_tool_message: bool = False, ) -> None: """ Extract text content and images from a message. Override this method to customize text/image extraction logic. """ - if skip_system_message and str(message.get("role") or "").lower() == "system": + role = str(message.get("role") or "").lower() + if skip_system_message and role == "system": + return + if skip_tool_message and role == "tool": return content = message.get("content", None) diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index cdd2d775371..97ece6b5eab 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -14,7 +14,22 @@ def effective_skip_system_message_for_guardrail(guardrail_to_apply: Any) -> bool return bool(getattr(litellm, "skip_system_message_in_guardrail", False)) +def effective_skip_tool_message_for_guardrail(guardrail_to_apply: Any) -> bool: + per = getattr(guardrail_to_apply, "skip_tool_message_in_guardrail", None) + if per is not None: + return bool(per) + import litellm + + return bool(getattr(litellm, "skip_tool_message_in_guardrail", False)) + + def openai_messages_without_system( messages: List[AllMessageValues], ) -> List[AllMessageValues]: return [m for m in messages if str((m or {}).get("role") or "").lower() != "system"] + + +def openai_messages_without_tool( + messages: List[AllMessageValues], +) -> List[AllMessageValues]: + return [m for m in messages if str((m or {}).get("role") or "").lower() != "tool"] diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 86ca6625629..d413a244539 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -21,7 +21,9 @@ from litellm._logging import verbose_proxy_logger from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, openai_messages_without_system, + openai_messages_without_tool, ) from litellm.main import stream_chunk_builder from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam @@ -73,6 +75,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return data skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply) + skip_tool = effective_skip_tool_message_for_guardrail(guardrail_to_apply) texts_to_check: List[str] = [] images_to_check: List[str] = [] @@ -91,6 +94,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): text_task_mappings=text_task_mappings, tool_call_task_mappings=tool_call_task_mappings, skip_system_message=skip_system, + skip_tool_message=skip_tool, ) # Step 2: Apply guardrail to all texts and tool calls in batch @@ -102,11 +106,15 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs["tool_calls"] = tool_calls_to_check # type: ignore structured_messages = self.get_structured_messages(data) if structured_messages: - inputs["structured_messages"] = ( - openai_messages_without_system(structured_messages) - if skip_system - else structured_messages - ) + if skip_system: + structured_messages = openai_messages_without_system( + structured_messages + ) + if skip_tool: + structured_messages = openai_messages_without_tool( + structured_messages + ) + inputs["structured_messages"] = structured_messages # Pass tools (function definitions) to the guardrail tools = data.get("tools") if tools: @@ -176,13 +184,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation): text_task_mappings: List[Tuple[int, Optional[int]]], tool_call_task_mappings: List[Tuple[int, int]], skip_system_message: bool = False, + skip_tool_message: bool = False, ) -> None: """ Extract text content, images, and tool calls from a message. Override this method to customize text/image/tool call extraction logic. """ - if skip_system_message and str(message.get("role") or "").lower() == "system": + role = str(message.get("role") or "").lower() + if skip_system_message and role == "system": + return + if skip_tool_message and role == "tool": return content = message.get("content", None) diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 868b23756d2..838fb2e01ad 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -482,6 +482,11 @@ class InMemoryGuardrailHandler: "skip_system_message_in_guardrail", getattr(litellm_params, "skip_system_message_in_guardrail", None), ) + setattr( + custom_guardrail_callback, + "skip_tool_message_in_guardrail", + getattr(litellm_params, "skip_tool_message_in_guardrail", None), + ) parsed_guardrail = Guardrail( guardrail_id=guardrail.get("guardrail_id"), diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 04347aebe3b..751113400d3 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -633,6 +633,16 @@ class BaseLitellmParams( ), ) + skip_tool_message_in_guardrail: Optional[bool] = Field( + default=None, + description=( + "When True, unified guardrails skip tool-role messages when building " + "evaluation inputs (texts and structured_messages). When False, tool " + "messages are included even if litellm_settings sets a global skip. When " + "None, use the global litellm.skip_tool_message_in_guardrail setting." + ), + ) + # Lakera specific params category_thresholds: Optional[LakeraCategoryThresholds] = Field( default=None, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 2418d7af04b..6e027fa4941 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -8,7 +8,9 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, openai_messages_without_system, + openai_messages_without_tool, ) from litellm.llms.openai.chat.guardrail_translation.handler import ( OpenAIChatCompletionsHandler, @@ -180,6 +182,136 @@ class TestUnifiedLLMGuardrails: } assert "system" in roles + class TestSkipToolMessageForChatCompletions: + def test_openai_messages_without_tool(self): + msgs = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "content": "tool result", "tool_call_id": "call_1"}, + ] + out = openai_messages_without_tool(msgs) + assert len(out) == 2 + assert all(m["role"] != "tool" for m in out) + assert msgs[2]["content"] == "tool result" + + def test_effective_skip_tool_respects_per_guardrail_over_global( + self, monkeypatch + ): + monkeypatch.setattr( + litellm, "skip_tool_message_in_guardrail", True, raising=False + ) + + class G: + skip_tool_message_in_guardrail = False + + assert effective_skip_tool_message_for_guardrail(G()) is False + + class G2: + skip_tool_message_in_guardrail = None + + assert effective_skip_tool_message_for_guardrail(G2()) is True + + @pytest.mark.asyncio + async def test_openai_handler_skips_tool_in_guardrail_inputs(self, monkeypatch): + monkeypatch.setattr( + litellm, "skip_tool_message_in_guardrail", True, raising=False + ) + + captured = {} + + class MockGuardrail: + skip_tool_message_in_guardrail = None + + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ): + captured["inputs"] = inputs + return inputs + + data = { + "messages": [ + {"role": "user", "content": "hello"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + } + ], + }, + { + "role": "tool", + "content": "secret tool result", + "tool_call_id": "call_1", + }, + ], + "model": "gpt-4o", + } + + handler = OpenAIChatCompletionsHandler() + await handler.process_input_messages( + data=data, + guardrail_to_apply=MockGuardrail(), + litellm_logging_obj=None, + ) + + assert "secret tool result" not in captured["inputs"]["texts"] + sm = captured["inputs"].get("structured_messages") or [] + assert all(m.get("role") != "tool" for m in sm) + assert data["messages"][2]["content"] == "secret tool result" + + @pytest.mark.asyncio + async def test_openai_handler_per_guardrail_skip_tool_false_overrides_global( + self, monkeypatch + ): + monkeypatch.setattr( + litellm, "skip_tool_message_in_guardrail", True, raising=False + ) + + captured = {} + + class MockGuardrail: + skip_tool_message_in_guardrail = False + + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ): + captured["inputs"] = inputs + return inputs + + data = { + "messages": [ + {"role": "user", "content": "u"}, + {"role": "tool", "content": "tr", "tool_call_id": "call_1"}, + ], + } + + await OpenAIChatCompletionsHandler().process_input_messages( + data=data, + guardrail_to_apply=MockGuardrail(), + litellm_logging_obj=None, + ) + + assert "tr" in captured["inputs"]["texts"] + roles = { + m.get("role") + for m in (captured["inputs"].get("structured_messages") or []) + } + assert "tool" in roles + class TestAsyncPreCallHook: @pytest.mark.asyncio async def test_uses_mcp_event_type(self): From bdb2b0e708a032d6235b934917d297168f7bd9e1 Mon Sep 17 00:00:00 2001 From: shivam Date: Thu, 7 May 2026 19:11:41 -0700 Subject: [PATCH 07/23] feat(dashboard): skip_tool_message_in_guardrail in guardrail UI Adds a tri-state control (inherit / yes / no) when creating or editing guardrails so admins can set litellm_params.skip_tool_message_in_guardrail without YAML, mirroring the existing skip_system_message control. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../guardrails/add_guardrail_form.tsx | 20 +++++++++++ .../guardrails/edit_guardrail_form.tsx | 23 ++++++++++++ .../components/guardrails/guardrail_info.tsx | 36 +++++++++++++++++++ .../guardrail_info_helpers.test.tsx | 16 +++++++++ .../guardrails/guardrail_info_helpers.tsx | 16 +++++++++ .../components/guardrails/guardrail_table.tsx | 10 +++++- 6 files changed, 120 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx index c91a6e85cdd..16c1c6efecd 100644 --- a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx @@ -5,6 +5,7 @@ import { createGuardrailCall, getGuardrailProviderSpecificParams, getGuardrailUI import ContentFilterConfiguration from "./content_filter/ContentFilterConfiguration"; import { choiceToSkipSystemForCreate, + choiceToSkipToolForCreate, getGuardrailProviders, guardrail_provider_map, guardrailLogoMap, @@ -188,6 +189,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a mode: preset.mode, default_on: preset.defaultOn, skip_system_message_choice: "inherit", + skip_tool_message_choice: "inherit", }; if (preset.provider === "BlockCodeExecution") { baseValues.confidence_threshold = 0.5; @@ -433,6 +435,11 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a guardrailData.litellm_params.skip_system_message_in_guardrail = skipForCreate; } + const skipToolForCreate = choiceToSkipToolForCreate(values.skip_tool_message_choice); + if (skipToolForCreate !== undefined) { + guardrailData.litellm_params.skip_tool_message_in_guardrail = skipToolForCreate; + } + // For Presidio PII, add the entity and action configurations if (values.provider === "PresidioPII" && selectedEntities.length > 0) { const piiEntitiesConfig: { [key: string]: string } = {}; @@ -804,6 +811,18 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a + + + + {/* Use the GuardrailProviderFields component to render provider-specific fields */} {!isToolPermissionProvider && !shouldRenderContentFilterConfigSettings(selectedProvider) && !shouldRenderLLMJudgeFields(selectedProvider) && ( = ({ visible, onClose, a mode: "pre_call", default_on: false, skip_system_message_choice: "inherit", + skip_tool_message_choice: "inherit", }} > {stepConfigs.map((step, index) => { diff --git a/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx b/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx index ad823df53fc..8ba9b0b312f 100644 --- a/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx @@ -6,6 +6,7 @@ import { guardrailLogoMap, getGuardrailProviders, type SkipSystemMessageChoice, + type SkipToolMessageChoice, } from "./guardrail_info_helpers"; import { getGuardrailUISettings, getGlobalLitellmHeaderName } from "../networking"; import PiiConfiguration from "./pii_configuration"; @@ -29,6 +30,7 @@ interface EditGuardrailFormProps { default_on: boolean; pii_entities_config?: { [key: string]: string }; skip_system_message_choice?: SkipSystemMessageChoice; + skip_tool_message_choice?: SkipToolMessageChoice; [key: string]: any; }; } @@ -138,6 +140,15 @@ const EditGuardrailForm: React.FC = ({ delete litellm_params.skip_system_message_in_guardrail; } + const skipToolChoice = values.skip_tool_message_choice as SkipToolMessageChoice | undefined; + if (skipToolChoice === "yes") { + litellm_params.skip_tool_message_in_guardrail = true; + } else if (skipToolChoice === "no") { + litellm_params.skip_tool_message_in_guardrail = false; + } else { + delete litellm_params.skip_tool_message_in_guardrail; + } + let guardrail_info: any = {}; // For Presidio PII, add the entity and action configurations @@ -432,6 +443,18 @@ const EditGuardrailForm: React.FC = ({ + + + + {renderProviderSpecificFields()}
diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx index 60400443d5c..53aebcff0de 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx @@ -29,7 +29,9 @@ import { getGuardrailLogoAndName, guardrail_provider_map, skipSystemMessageToChoice, + skipToolMessageToChoice, type SkipSystemMessageChoice, + type SkipToolMessageChoice, } from "./guardrail_info_helpers"; import GuardrailOptionalParams from "./guardrail_optional_params"; import GuardrailProviderFields from "./guardrail_provider_fields"; @@ -214,12 +216,16 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, if (guardrailData && form) { const lp = { ...(guardrailData.litellm_params || {}) }; delete lp.skip_system_message_in_guardrail; + delete lp.skip_tool_message_in_guardrail; form.setFieldsValue({ guardrail_name: guardrailData.guardrail_name, ...lp, skip_system_message_choice: skipSystemMessageToChoice( guardrailData.litellm_params?.skip_system_message_in_guardrail, ), + skip_tool_message_choice: skipToolMessageToChoice( + guardrailData.litellm_params?.skip_tool_message_in_guardrail, + ), guardrail_info: guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : "", // Include any optional_params if they exist ...(guardrailData.litellm_params?.optional_params && { @@ -302,6 +308,20 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, } } + const prevSkipToolChoice = skipToolMessageToChoice( + guardrailData.litellm_params?.skip_tool_message_in_guardrail, + ); + const nextSkipToolChoice = values.skip_tool_message_choice as SkipToolMessageChoice | undefined; + if (nextSkipToolChoice !== undefined && nextSkipToolChoice !== prevSkipToolChoice) { + if (nextSkipToolChoice === "inherit") { + updateData.litellm_params.skip_tool_message_in_guardrail = null; + } else if (nextSkipToolChoice === "yes") { + updateData.litellm_params.skip_tool_message_in_guardrail = true; + } else { + updateData.litellm_params.skip_tool_message_in_guardrail = false; + } + } + // Only include guardrail_info if it has changed const originalGuardrailInfo = guardrailData.guardrail_info; const newGuardrailInfo = values.guardrail_info ? JSON.parse(values.guardrail_info) : undefined; @@ -674,11 +694,15 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, ...(() => { const lp = { ...(guardrailData.litellm_params || {}) }; delete lp.skip_system_message_in_guardrail; + delete lp.skip_tool_message_in_guardrail; return lp; })(), skip_system_message_choice: skipSystemMessageToChoice( guardrailData.litellm_params?.skip_system_message_in_guardrail, ), + skip_tool_message_choice: skipToolMessageToChoice( + guardrailData.litellm_params?.skip_tool_message_in_guardrail, + ), guardrail_info: guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : "", @@ -716,6 +740,18 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, + + + + {guardrailData.litellm_params?.guardrail === "presidio" && ( <> PII Protection diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx index dfda86c1e4a..1fc62f94cf1 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx @@ -12,6 +12,8 @@ import { GuardrailProviders, skipSystemMessageToChoice, choiceToSkipSystemForCreate, + skipToolMessageToChoice, + choiceToSkipToolForCreate, } from "./guardrail_info_helpers"; describe("guardrail_info_helpers", () => { @@ -215,4 +217,18 @@ describe("guardrail_info_helpers", () => { expect(choiceToSkipSystemForCreate("no")).toBe(false); }); }); + + describe("skipToolMessageToChoice / choiceToSkipToolForCreate", () => { + it("maps API values to form choices and back for create", () => { + expect(skipToolMessageToChoice(undefined)).toBe("inherit"); + expect(skipToolMessageToChoice(null)).toBe("inherit"); + expect(skipToolMessageToChoice(true)).toBe("yes"); + expect(skipToolMessageToChoice(false)).toBe("no"); + + expect(choiceToSkipToolForCreate("inherit")).toBeUndefined(); + expect(choiceToSkipToolForCreate(undefined)).toBeUndefined(); + expect(choiceToSkipToolForCreate("yes")).toBe(true); + expect(choiceToSkipToolForCreate("no")).toBe(false); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx index ac4b787e96a..54b16b81765 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx @@ -179,3 +179,19 @@ export function choiceToSkipSystemForCreate(choice: SkipSystemMessageChoice | un if (choice === "no") return false; return undefined; } + +/** Tri-state UI value for `litellm_params.skip_tool_message_in_guardrail` (inherit = use global). */ +export type SkipToolMessageChoice = "inherit" | "yes" | "no"; + +export function skipToolMessageToChoice(v: boolean | null | undefined): SkipToolMessageChoice { + if (v === true) return "yes"; + if (v === false) return "no"; + return "inherit"; +} + +/** Create flow: omit key when inheriting global default. */ +export function choiceToSkipToolForCreate(choice: SkipToolMessageChoice | undefined): boolean | undefined { + if (choice === "yes") return true; + if (choice === "no") return false; + return undefined; +} diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx index 5bb2da78fa2..ecf6ce48fde 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx @@ -11,7 +11,12 @@ import { SortingState, useReactTable, } from "@tanstack/react-table"; -import { getGuardrailLogoAndName, guardrail_provider_map, skipSystemMessageToChoice } from "./guardrail_info_helpers"; +import { + getGuardrailLogoAndName, + guardrail_provider_map, + skipSystemMessageToChoice, + skipToolMessageToChoice, +} from "./guardrail_info_helpers"; import EditGuardrailForm from "./edit_guardrail_form"; import { Guardrail, GuardrailDefinitionLocation } from "./types"; @@ -304,6 +309,9 @@ const GuardrailTable: React.FC = ({ skip_system_message_choice: skipSystemMessageToChoice( selectedGuardrail.litellm_params?.skip_system_message_in_guardrail, ), + skip_tool_message_choice: skipToolMessageToChoice( + selectedGuardrail.litellm_params?.skip_tool_message_in_guardrail, + ), ...selectedGuardrail.guardrail_info, }} /> From 144279eb57edb6cc0a97ad47c9da33b910f70dfa Mon Sep 17 00:00:00 2001 From: Parijat Sharma Date: Fri, 8 May 2026 23:16:46 +0530 Subject: [PATCH 08/23] fix(ui): URL-encode team_id in teamInfoCall to handle special characters (#27466) --- .../src/components/networking.test.ts | 49 +++++++++++++++++++ .../src/components/networking.tsx | 2 +- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/networking.test.ts b/ui/litellm-dashboard/src/components/networking.test.ts index 3c107fa586f..05aa2d51624 100644 --- a/ui/litellm-dashboard/src/components/networking.test.ts +++ b/ui/litellm-dashboard/src/components/networking.test.ts @@ -404,3 +404,52 @@ describe("individualModelHealthCheckCall", () => { expect(parsed.searchParams.get("model_id")).toBe("id/with/slashes"); }); }); + +describe("teamInfoCall", () => { + const originalFetch = global.fetch; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + global.fetch = originalFetch; + }); + + it("should URL-encode team_id query param to handle special characters safely", async () => { + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + json: vi.fn().mockResolvedValue({ team_id: "team with spaces & special?chars" }), + } as any); + global.fetch = mockFetch as any; + + const teamID = "team with spaces & special?chars"; + await Networking.teamInfoCall("token", teamID); + + expect(mockFetch).toHaveBeenCalledOnce(); + const [url] = mockFetch.mock.calls[0]; + const urlStr = typeof url === "string" ? url : (url as Request).url; + const parsed = typeof url === "string" ? new URL(url, "http://example.com") : new URL((url as Request).url); + + expect(urlStr).toContain("/team/info"); + // Encoded value is present in the raw URL string (verifies encodeURIComponent was used) + expect(urlStr).toContain(`team_id=${encodeURIComponent(teamID)}`); + // Round-trip parse returns the original team_id + expect(parsed.searchParams.get("team_id")).toBe(teamID); + }); + + it("should not append team_id when teamID is null", async () => { + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + json: vi.fn().mockResolvedValue({}), + } as any); + global.fetch = mockFetch as any; + + await Networking.teamInfoCall("token", null); + + expect(mockFetch).toHaveBeenCalledOnce(); + const [url] = mockFetch.mock.calls[0]; + const parsed = typeof url === "string" ? new URL(url, "http://example.com") : new URL((url as Request).url); + expect(parsed.searchParams.has("team_id")).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 44208904a70..8bfe7c98691 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -1386,7 +1386,7 @@ export const teamInfoCall = async (accessToken: string, teamID: string | null) = try { let url = proxyBaseUrl ? `${proxyBaseUrl}/team/info` : `/team/info`; if (teamID) { - url = `${url}?team_id=${teamID}`; + url = `${url}?team_id=${encodeURIComponent(teamID)}`; } console.log("in teamInfoCall"); const response = await fetch(url, { From ae67cecc22b069be64f6e4f197b3e8e5b1cfeb13 Mon Sep 17 00:00:00 2001 From: oss-agent-shin Date: Fri, 8 May 2026 15:30:41 -0700 Subject: [PATCH 09/23] Allow team admins to test model connections (#27487) Co-authored-by: oss-agent-shin <279349115+oss-agent-shin@users.noreply.github.com> Co-authored-by: ishaan-berri --- litellm/proxy/_types.py | 2 ++ .../proxy/auth/test_route_checks.py | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ed20fe86cdc..1c380392349 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -707,6 +707,8 @@ class LiteLLMRoutes(enum.Enum): # Project read routes - endpoint scopes results to caller's teams (non-admin) "/project/list", "/project/info", + # Endpoint enforces proxy-admin vs team-admin model access itself. + "/health/test_connection", # Invitation routes - org/team admins checked in endpoint via _user_has_admin_privileges "/invitation/new", "/invitation/delete", diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 268cdff1f7d..9c1cd116e69 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -80,6 +80,28 @@ def test_compliance_routes_open_to_internal_user(route): ) +def test_health_test_connection_route_delegates_internal_user_auth_to_endpoint(): + """Team model test-connection requests are authorized by the endpoint.""" + role = LitellmUserRoles.INTERNAL_USER.value + user_obj = LiteLLM_UserTable( + user_id="test_user", + user_email="test@example.com", + user_role=role, + ) + valid_token = UserAPIKeyAuth(user_id="test_user", user_role=role) + request = MagicMock(spec=Request) + request.query_params = {} + + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=role, + route="/health/test_connection", + request=request, + valid_token=valid_token, + request_data={}, + ) + + @pytest.mark.parametrize( "route", ["/compliance/eu-ai-act", "/compliance/gdpr"], From f2e97380d2deeeb8ab4b465f6cf5ee91d94233a9 Mon Sep 17 00:00:00 2001 From: oss-agent-shin Date: Fri, 8 May 2026 16:25:45 -0700 Subject: [PATCH 10/23] Add OpenRouter Qwen 3.6 Plus metadata (#27486) Co-authored-by: oss-agent-shin <279349115+oss-agent-shin@users.noreply.github.com> Co-authored-by: ishaan-berri --- ...odel_prices_and_context_window_backup.json | 14 ++++++++++++++ model_prices_and_context_window.json | 14 ++++++++++++++ tests/test_litellm/test_cost_calculator.py | 19 +++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4fba1980103..76d2d35a53a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -27187,6 +27187,20 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/qwen/qwen3.6-plus": { + "input_cost_per_token": 3.25e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.95e-06, + "source": "https://openrouter.ai/qwen/qwen3.6-plus", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "openrouter/qwen/qwen3.5-35b-a3b": { "input_cost_per_token": 2.5e-07, "litellm_provider": "openrouter", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 92e87c00ef6..e66ad8e0cf6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -27192,6 +27192,20 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/qwen/qwen3.6-plus": { + "input_cost_per_token": 3.25e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.95e-06, + "source": "https://openrouter.ai/qwen/qwen3.6-plus", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "openrouter/qwen/qwen3.5-35b-a3b": { "input_cost_per_token": 2.5e-07, "litellm_provider": "openrouter", diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index ebe175b2503..e53484dd287 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -110,6 +110,25 @@ def test_wandb_model_api_pricing_entries(): assert model_info["output_cost_per_token"] == output_cost +def test_openrouter_qwen36_plus_model_info(): + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model_info = litellm.model_cost.get("openrouter/qwen/qwen3.6-plus") + + assert model_info is not None + assert model_info["litellm_provider"] == "openrouter" + assert model_info["mode"] == "chat" + assert model_info["max_input_tokens"] == 1000000 + assert model_info["max_output_tokens"] == 65536 + assert model_info["input_cost_per_token"] == 3.25e-07 + assert model_info["output_cost_per_token"] == 1.95e-06 + assert model_info["supports_function_calling"] is True + assert model_info["supports_tool_choice"] is True + assert model_info["supports_reasoning"] is True + assert model_info["supports_vision"] is True + + def test_cost_calculator_with_usage(monkeypatch): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") From 13a193367f0ae2e7e8b29913e0d23a32d3bb0457 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 8 May 2026 17:39:39 -0700 Subject: [PATCH 11/23] feat(sso): show full IdP claims in /sso/debug/callback (#27498) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(sso): show full IdP claims in /sso/debug/callback The debug callback only displayed the proxy-parsed OpenID summary, so customers couldn't verify what custom claims (team_id, team_alias, roles, etc.) the IdP was actually returning. Render two new sections — Raw Claims (userinfo) and Access Token Claims (decoded JWT) — alongside the existing parsed view. Strip bearer tokens defense-in-depth in case a non-conforming IdP places them in its userinfo response. Resolves LIT-2838 * Update litellm/proxy/management_endpoints/ui_sso.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * fix(sso): hoist json.dumps out of f-string for py3.10 ruff --------- Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../html_forms/jwt_display_template.py | 203 ++++++++++++------ litellm/proxy/management_endpoints/ui_sso.py | 42 +++- .../proxy/management_endpoints/test_ui_sso.py | 160 ++++++++++++++ 3 files changed, 330 insertions(+), 75 deletions(-) diff --git a/litellm/proxy/common_utils/html_forms/jwt_display_template.py b/litellm/proxy/common_utils/html_forms/jwt_display_template.py index 03dff78dba0..ea65fd0e28f 100644 --- a/litellm/proxy/common_utils/html_forms/jwt_display_template.py +++ b/litellm/proxy/common_utils/html_forms/jwt_display_template.py @@ -14,7 +14,7 @@ jwt_display_template = """ padding: 20px; display: flex; justify-content: center; - align-items: center; + align-items: flex-start; min-height: 100vh; color: #333; } @@ -27,18 +27,18 @@ jwt_display_template = """ width: 800px; max-width: 100%; } - + .logo-container { text-align: center; margin-bottom: 30px; } - + .logo { font-size: 24px; font-weight: 600; color: #1e293b; } - + h2 { margin: 0 0 10px; color: #1e293b; @@ -46,7 +46,14 @@ jwt_display_template = """ font-weight: 600; text-align: center; } - + + h3 { + margin: 0 0 12px; + color: #1e293b; + font-size: 18px; + font-weight: 600; + } + .subtitle { color: #64748b; margin: 0 0 20px; @@ -58,15 +65,15 @@ jwt_display_template = """ background-color: #f1f5f9; border-radius: 6px; padding: 20px; - margin-bottom: 30px; + margin-bottom: 20px; border-left: 4px solid #2563eb; } - + .success-box { background-color: #f0fdf4; border-radius: 6px; padding: 20px; - margin-bottom: 30px; + margin-bottom: 20px; border-left: 4px solid #16a34a; } @@ -78,7 +85,7 @@ jwt_display_template = """ font-weight: 600; font-size: 16px; } - + .success-header { display: flex; align-items: center; @@ -87,46 +94,53 @@ jwt_display_template = """ font-weight: 600; font-size: 16px; } - + .info-header svg, .success-header svg { margin-right: 8px; } - + .data-container { margin-top: 20px; } - + .data-row { display: flex; border-bottom: 1px solid #e2e8f0; padding: 12px 0; } - + .data-row:last-child { border-bottom: none; } - + .data-label { font-weight: 500; color: #334155; - width: 180px; + width: 220px; flex-shrink: 0; } - + .data-value { color: #475569; word-break: break-all; } - + + .empty-note { + color: #64748b; + font-style: italic; + margin: 0; + font-size: 14px; + } + .jwt-container { background-color: #f8fafc; border-radius: 6px; padding: 15px; - margin-top: 20px; + margin-top: 12px; overflow-x: auto; border: 1px solid #e2e8f0; } - + .jwt-text { font-family: monospace; white-space: pre-wrap; @@ -134,7 +148,7 @@ jwt_display_template = """ margin: 0; color: #334155; } - + .back-button { display: inline-block; background-color: #6466E9; @@ -146,18 +160,18 @@ jwt_display_template = """ margin-top: 20px; text-align: center; } - + .back-button:hover { background-color: #4138C2; text-decoration: none; } - + .buttons { display: flex; gap: 10px; - margin-top: 20px; + margin-top: 12px; } - + .copy-button { background-color: #e2e8f0; color: #334155; @@ -169,11 +183,11 @@ jwt_display_template = """ display: flex; align-items: center; } - + .copy-button:hover { background-color: #cbd5e1; } - + .copy-button svg { margin-right: 6px; } @@ -188,7 +202,7 @@ jwt_display_template = """

SSO Debug Information

Results from the SSO authentication process.

- +
@@ -199,11 +213,7 @@ jwt_display_template = """

The SSO authentication completed successfully. Below is the information returned by the provider.

- -
- -
- +
@@ -211,22 +221,62 @@ jwt_display_template = """ - JSON Representation + Parsed by Proxy
+

Fields the proxy extracted into its internal user model.

+
+ +
+
+ +
+
+ + + + + + Raw Claims (userinfo) +
+

Complete set of claims returned by the IdP's userinfo endpoint.

- 
Loading...
+ 
Loading...
-
- + +
+
+ + + + + + Access Token Claims +
+

Decoded payload of the access token JWT (when the IdP issues one).

+
+ 
Loading...
+
+
+ +
+
+ Try Another SSO Login @@ -234,39 +284,58 @@ jwt_display_template = """ diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 74ee7c7220d..b13118db6fe 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -4078,6 +4078,8 @@ async def debug_sso_callback(request: Request): redirect_url += "/sso/debug/callback" result = None + received_response: Optional[dict] = None + access_token_payload: Optional[dict] = None if google_client_id is not None: result = await GoogleSSOHandler.get_google_callback_response( request=request, @@ -4094,12 +4096,14 @@ async def debug_sso_callback(request: Request): ) elif generic_client_id is not None: - result, _, _ = await get_generic_sso_response( - request=request, - jwt_handler=jwt_handler, - generic_client_id=generic_client_id, - redirect_url=redirect_url, - sso_jwt_handler=sso_jwt_handler, + result, received_response, access_token_payload = ( + await get_generic_sso_response( + request=request, + jwt_handler=jwt_handler, + generic_client_id=generic_client_id, + redirect_url=redirect_url, + sso_jwt_handler=sso_jwt_handler, + ) ) # If result is None, return a basic error message @@ -4128,10 +4132,32 @@ async def debug_sso_callback(request: Request): except Exception as e: filtered_result[key] = f"Complex value (not displayable): {str(e)}" + # Defense-in-depth: ensure no bearer tokens leak into the rendered HTML even if + # a non-conforming IdP places them in its userinfo response. + safe_raw_claims = { + k: v + for k, v in (received_response or {}).items() + if k not in _OAUTH_TOKEN_FIELDS + } + safe_access_token_claims = { + k: v + for k, v in (access_token_payload or {}).items() + if k not in _OAUTH_TOKEN_FIELDS + } + + sso_payload = { + "parsed_by_proxy": filtered_result, + "raw_claims": safe_raw_claims, + "access_token_claims": safe_access_token_claims, + } + # Replace the placeholder in the template with the actual data + sso_payload_json = json.dumps(sso_payload, indent=2, default=str).replace( + " Date: Fri, 8 May 2026 18:09:14 -0700 Subject: [PATCH 12/23] fix(proxy): point /metrics 401 at the opt-out flag Operators upgrading past 35bbca60b0 (which made /metrics auth default-on) see "Malformed API Key passed in. Ensure Key has 'Bearer ' prefix." with no hint that litellm_settings.require_auth_for_metrics_endpoint: false restores the previous unauthenticated behavior. Append that discovery hint to the existing 401 body so a Prometheus scraper that breaks after upgrade has a clear migration path. No behavior change. --- .../middleware/prometheus_auth_middleware.py | 5 ++++- .../test_prometheus_auth_middleware.py | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/middleware/prometheus_auth_middleware.py b/litellm/proxy/middleware/prometheus_auth_middleware.py index cfc4cbd64b2..7eb8ae83cb4 100644 --- a/litellm/proxy/middleware/prometheus_auth_middleware.py +++ b/litellm/proxy/middleware/prometheus_auth_middleware.py @@ -79,7 +79,10 @@ class PrometheusAuthMiddleware: # Send 401 response directly via ASGI protocol error_message = getattr(e, "message", str(e)) body = json.dumps( - f"Unauthorized access to metrics endpoint: {error_message}" + f"Unauthorized access to metrics endpoint: {error_message} " + f"To allow unauthenticated access, set " + f"`litellm_settings.require_auth_for_metrics_endpoint: false` " + f"in your proxy_config.yaml." ).encode("utf-8") await send( { diff --git a/tests/test_litellm/proxy/middleware/test_prometheus_auth_middleware.py b/tests/test_litellm/proxy/middleware/test_prometheus_auth_middleware.py index 6cab5baee9a..1d0c0f90fd1 100644 --- a/tests/test_litellm/proxy/middleware/test_prometheus_auth_middleware.py +++ b/tests/test_litellm/proxy/middleware/test_prometheus_auth_middleware.py @@ -121,6 +121,26 @@ def test_invalid_auth_metrics(app_with_middleware, monkeypatch): assert "Unauthorized access to metrics endpoint" in response.text +def test_invalid_auth_metrics_includes_optout_hint(app_with_middleware, monkeypatch): + """ + The 401 body must tell operators how to restore the previous unauthenticated + behavior, otherwise a Prometheus scraper that worked pre-upgrade just sees + "Malformed API Key" with no actionable migration path. + """ + monkeypatch.setattr(litellm, "require_auth_for_metrics_endpoint", True) + monkeypatch.setattr( + "litellm.proxy.middleware.prometheus_auth_middleware.user_api_key_auth", + fake_invalid_auth, + ) + + client = TestClient(app_with_middleware) + response = client.get("/metrics") + + assert response.status_code == 401, response.text + assert "require_auth_for_metrics_endpoint" in response.text + assert "false" in response.text + + def test_metrics_auth_uses_real_auth_when_route_is_public( app_with_middleware, monkeypatch ): From adc41ade8c0eb15ec6165906b2c3c9803b26d386 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 8 May 2026 20:18:31 -0700 Subject: [PATCH 13/23] fix(proxy): bound budget reservation per request instead of pinning to remaining headroom reserve_budget_for_request fell back to reserving the entire remaining team/key/user headroom whenever a request omitted max_tokens, which pinned the spend counter at max_budget for the duration of the in-flight request and false-positive-blocked every concurrent or back-to-back request until the success callback reconciled. Surfaced as an integration-test team being budget-blocked at its $2000 cap while DB spend was $0.144. Switch the missing-max_tokens path to a fixed default of 16384 output tokens (mirrors parallel_request_limiter_v3's DEFAULT_MAX_TOKENS_ESTIMATE precedent), and clamp explicit max_tokens at the model's max_output_tokens for reservation accounting only. The outbound request body is unchanged, so providers see whatever the caller actually sent; only the local integer used to compute reservation cost is bounded. This also prevents a hostile max_tokens=999999999 from inflating one request's reservation up to the entire team headroom. For Opus 4.7 (output $25/M, max_output 128K) on a $2000 budget the worst-case per-request reservation drops from "everything left" to $3.20, raising admittable concurrency from 1 to ~625. --- .../spend_tracking/budget_reservation.py | 63 ++-- .../proxy/test_budget_reservation.py | 327 ++++++++---------- 2 files changed, 164 insertions(+), 226 deletions(-) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 1d296611bfc..feae368d23f 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -95,11 +95,9 @@ async def reserve_budget_for_request( route=route, llm_router=llm_router, ) - if reservation_cost is None: - reservation_cost = await _get_smallest_remaining_budget( - counters=counters, - current_spend_by_counter_key=current_spend_by_counter_key, - ) + # estimate_request_max_cost still returns None when the model is unknown + # to the cost map (no token-priced cost fields, e.g. image/audio routes). + # In that case we fall back to read-time enforcement only. if reservation_cost is None or reservation_cost <= 0: return None @@ -553,32 +551,6 @@ def _coerce_window(window: Any) -> dict: return {} -async def _get_smallest_remaining_budget( - counters: List[_BudgetCounter], - current_spend_by_counter_key: Dict[str, float], -) -> Optional[float]: - remaining_budget: Optional[float] = None - for counter in counters: - current_spend = await _get_current_counter_value(counter=counter) - current_spend_by_counter_key[counter.counter_key] = current_spend - remaining = counter.max_budget - current_spend - if remaining <= 0: - raise litellm.BudgetExceededError( - current_cost=current_spend, - max_budget=counter.max_budget, - message=( - "Budget has been exceeded! " - f"{counter.entity_type}={counter.entity_id} " - f"Current cost: {current_spend}, " - f"Max budget: {counter.max_budget}" - ), - ) - remaining_budget = ( - remaining if remaining_budget is None else min(remaining_budget, remaining) - ) - return remaining_budget - - async def _reserve_counter( counter: _BudgetCounter, reservation_cost: float, @@ -946,6 +918,9 @@ def _estimate_input_tokens( return None +DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK = 16384 + + def _estimate_output_tokens( request_body: dict, route: str, @@ -954,15 +929,27 @@ def _estimate_output_tokens( if _is_input_only_route(route=route): return 0 + requested: Optional[int] = None for key in ("max_completion_tokens", "max_tokens", "max_output_tokens"): - max_tokens = _to_int(request_body.get(key)) - if max_tokens is not None: - return max_tokens + requested = _to_int(request_body.get(key)) + if requested is not None: + break - # If the caller did not cap output tokens, avoid reserving a model's - # theoretical maximum context. The caller can still admit one request by - # reserving the smallest remaining budget in reserve_budget_for_request(). - return None + # Clamp at min(requested-or-default, model_max-or-default). Two purposes: + # (1) Without an explicit cap we still need a finite reservation so the + # atomic admission counter actually bounds concurrent in-flight cost + # (mirrors parallel_request_limiter_v3's DEFAULT_MAX_TOKENS_ESTIMATE). + # (2) An adversarial caller cannot send max_tokens=999999999 to inflate + # the reservation up to remaining team headroom and pin the counter + # at the cap — the model can only physically emit max_output_tokens + # anyway, so reserving more is both wasteful and a DoS surface. + model_ceiling = ( + _to_int(model_info.get("max_output_tokens")) + or DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK + ) + if requested is None: + requested = DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK + return min(requested, model_ceiling) def _count_text_tokens(model: str, text: Any) -> int: diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 070b232066a..0e2ca98a113 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -585,15 +585,24 @@ async def test_should_cap_known_estimate_to_remaining_budget( @pytest.mark.asyncio -async def test_should_reserve_remaining_budget_when_output_cap_missing( +async def test_should_clamp_reservation_to_default_when_output_cap_missing( spend_counter_state, ): + """When max_tokens is not specified, _estimate_output_tokens falls back to + DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK (16K), clamped by the model's + max_output_tokens. Reservation must be a bounded per-request amount + (mirroring parallel_request_limiter_v3's DEFAULT_MAX_TOKENS_ESTIMATE), + not the entire remaining headroom.""" + from litellm.proxy.spend_tracking.budget_reservation import ( + DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK, + ) + counter_cache, key_cache = spend_counter_state proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) valid_token = UserAPIKeyAuth( token="key-budget-uncapped", spend=0.2, - max_budget=1.0, + max_budget=10000.0, ) await key_cache.async_set_cache( key="key-budget-uncapped", @@ -602,22 +611,24 @@ async def test_should_reserve_remaining_budget_when_output_cap_missing( request_body = _request_body() request_body.pop("max_tokens") + output_cost_per_token = 1e-5 # roughly Opus 4.5/4.7 output rate + expected_cost = DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK * output_cost_per_token + with patch( "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", return_value={ "input_cost_per_token": 0.0, - "output_cost_per_token": 100.0, - "max_output_tokens": 200000, + "output_cost_per_token": output_cost_per_token, + "max_output_tokens": 200000, # well above the 16K fallback }, ): - assert ( - estimate_request_max_cost( - request_body=request_body, - route="/chat/completions", - llm_router=None, - ) - is None + estimated = estimate_request_max_cost( + request_body=request_body, + route="/chat/completions", + llm_router=None, ) + assert estimated == pytest.approx(expected_cost) + reservation = await reserve_budget_for_request( request_body=request_body, route="/chat/completions", @@ -631,47 +642,45 @@ async def test_should_reserve_remaining_budget_when_output_cap_missing( ) assert reservation is not None - assert reservation["reserved_cost"] == pytest.approx(0.8) - assert counter_cache.in_memory_cache.get_cache( - key="spend:key:key-budget-uncapped" - ) == pytest.approx(1.0) - + assert reservation["reserved_cost"] == pytest.approx(expected_cost) await release_budget_reservation(reservation) @pytest.mark.asyncio -async def test_should_shrink_uncapped_reservation_when_counter_advances( +async def test_should_clamp_reservation_to_model_ceiling_when_caller_overrequests( spend_counter_state, - monkeypatch, ): + """An adversarial caller sending max_tokens=999_999_999 must not be able + to inflate the per-request reservation up to the entire remaining team + headroom. _estimate_output_tokens clamps the explicit value at the + model's max_output_tokens — the model can only physically emit that + many tokens anyway, so anything more is both wasteful and a DoS surface.""" counter_cache, key_cache = spend_counter_state proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) valid_token = UserAPIKeyAuth( - token="key-budget-uncapped-race", - spend=0.2, - max_budget=1.0, + token="key-budget-overrequest", + spend=0.0, + max_budget=10000.0, ) + await key_cache.async_set_cache( + key="key-budget-overrequest", + value=valid_token, + ) + request_body = _request_body() - request_body.pop("max_tokens") + request_body["max_tokens"] = 999_999_999 - from litellm.proxy.spend_tracking import budget_reservation - - async def stale_counter_read(counter): - await counter_cache.async_increment_cache( - key=counter.counter_key, - value=0.3, - ) - return 0.2 - - monkeypatch.setattr( - budget_reservation, - "_get_current_counter_value", - stale_counter_read, - ) + output_cost_per_token = 1e-5 + model_ceiling = 128_000 + expected_cost = model_ceiling * output_cost_per_token with patch( - "litellm.proxy.spend_tracking.budget_reservation.estimate_request_max_cost", - return_value=None, + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "input_cost_per_token": 0.0, + "output_cost_per_token": output_cost_per_token, + "max_output_tokens": model_ceiling, + }, ): reservation = await reserve_budget_for_request( request_body=request_body, @@ -686,101 +695,9 @@ async def test_should_shrink_uncapped_reservation_when_counter_advances( ) assert reservation is not None - assert reservation["reserved_cost"] == pytest.approx(0.7) - assert counter_cache.in_memory_cache.get_cache( - key="spend:key:key-budget-uncapped-race" - ) == pytest.approx(1.0) - + assert reservation["reserved_cost"] == pytest.approx(expected_cost) await release_budget_reservation(reservation) - assert counter_cache.in_memory_cache.get_cache( - key="spend:key:key-budget-uncapped-race" - ) == pytest.approx(0.3) - - -@pytest.mark.asyncio -async def test_should_shrink_uncapped_reservation_multiple_times( - spend_counter_state, - monkeypatch, -): - counter_cache, key_cache = spend_counter_state - proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) - valid_token = UserAPIKeyAuth( - token="key-budget-double-resize", - spend=0.2, - max_budget=1.0, - team_id="team-budget-double-resize", - ) - team_object = LiteLLM_TeamTable( - team_id="team-budget-double-resize", - spend=0.2, - max_budget=1.0, - ) - request_body = _request_body() - request_body.pop("max_tokens") - - from litellm.proxy.spend_tracking import budget_reservation - - stale_spend_by_counter_key = { - "spend:key:key-budget-double-resize": 0.3, - "spend:team:team-budget-double-resize": 0.4, - } - - async def stale_counter_read(counter): - await counter_cache.async_increment_cache( - key=counter.counter_key, - value=stale_spend_by_counter_key[counter.counter_key], - ) - return 0.2 - - monkeypatch.setattr( - budget_reservation, - "_get_current_counter_value", - stale_counter_read, - ) - - with patch( - "litellm.proxy.spend_tracking.budget_reservation.estimate_request_max_cost", - return_value=None, - ): - reservation = await reserve_budget_for_request( - request_body=request_body, - route="/chat/completions", - llm_router=None, - valid_token=valid_token, - team_object=team_object, - user_object=None, - prisma_client=None, - user_api_key_cache=key_cache, - proxy_logging_obj=proxy_logging_obj, - ) - - assert reservation is not None - assert reservation["reserved_cost"] == pytest.approx(0.6) - assert [entry["reserved_cost"] for entry in reservation["entries"]] == [ - pytest.approx(0.6), - pytest.approx(0.6), - ] - assert [entry["applied_adjustment"] for entry in reservation["entries"]] == [ - pytest.approx(0.0), - pytest.approx(0.0), - ] - assert counter_cache.in_memory_cache.get_cache( - key="spend:key:key-budget-double-resize" - ) == pytest.approx(0.9) - assert counter_cache.in_memory_cache.get_cache( - key="spend:team:team-budget-double-resize" - ) == pytest.approx(1.0) - - await release_budget_reservation(reservation) - - assert counter_cache.in_memory_cache.get_cache( - key="spend:key:key-budget-double-resize" - ) == pytest.approx(0.3) - assert counter_cache.in_memory_cache.get_cache( - key="spend:team:team-budget-double-resize" - ) == pytest.approx(0.4) - def test_should_start_window_without_reset_at_at_duration_boundary(): before = datetime.now(timezone.utc) - timedelta(hours=1) @@ -1047,62 +964,6 @@ async def test_should_release_tracked_entry_when_reservation_fails_after_increme ) == pytest.approx(0.0) -@pytest.mark.asyncio -async def test_should_not_re_read_uncapped_budget_after_reservation_fallback( - spend_counter_state, - monkeypatch, -): - _, key_cache = spend_counter_state - proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) - valid_token = UserAPIKeyAuth( - token="key-budget-uncapped-read-once", - spend=0.2, - max_budget=1.0, - ) - - from litellm.proxy.spend_tracking import budget_reservation - - current_counter_reads = [] - - async def mock_get_current_counter_value(counter): - current_counter_reads.append(counter.counter_key) - return counter.fallback_spend - - async def mock_reserve_counter(counter, reservation_cost): - return None - - monkeypatch.setattr( - budget_reservation, - "_get_current_counter_value", - mock_get_current_counter_value, - ) - monkeypatch.setattr( - budget_reservation, - "_reserve_counter", - mock_reserve_counter, - ) - - with patch( - "litellm.proxy.spend_tracking.budget_reservation.estimate_request_max_cost", - return_value=None, - ): - reservation = await reserve_budget_for_request( - request_body=_request_body(), - route="/chat/completions", - llm_router=None, - valid_token=valid_token, - team_object=None, - user_object=None, - prisma_client=None, - user_api_key_cache=key_cache, - proxy_logging_obj=proxy_logging_obj, - ) - - assert reservation is not None - assert reservation["reserved_cost"] == pytest.approx(0.8) - assert current_counter_reads == ["spend:key:key-budget-uncapped-read-once"] - - @pytest.mark.asyncio async def test_should_reconcile_reserved_counter_to_actual_spend( spend_counter_state, @@ -1492,4 +1353,94 @@ async def test_should_reserve_all_budgeted_counters(spend_counter_state): counter_cache.in_memory_cache.get_cache(key="spend:team:team-budget-all") == 0.3 ) - await release_budget_reservation(reservation) + +@pytest.mark.asyncio +async def test_should_not_block_concurrent_team_request_when_first_request_lacks_max_tokens( + spend_counter_state, +): + """ + Regression test: a team-bound request with no max_tokens must not pin the + team's spend counter at max_budget for the duration of the request. + + Repro of the integration-test team being falsely budget-blocked at the + $2000 cap while DB spend is $0.144: the first request without max_tokens + used to reserve the entire remaining headroom, leaving any subsequent + request stuck behind a counter sitting at the cap until the success + callback finished reconciling. + """ + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + + valid_token = UserAPIKeyAuth( + token="key-team-integration-tests", + spend=0.0, + team_id="team-integration-tests", + ) + team_object = LiteLLM_TeamTable( + team_id="team-integration-tests", + max_budget=2000.0, + spend=0.144, + ) + await key_cache.async_set_cache( + key=f"team_id:{team_object.team_id}", + value=team_object, + ) + + request_body = _request_body() + request_body.pop("max_tokens") + + # Realistic Opus 4.7 output pricing — the 16K fallback × $25/M ≈ $0.40 + # reservation per request, leaving ~5000 admittable concurrent requests + # against a $2000 team budget. + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "input_cost_per_token": 5e-6, + "output_cost_per_token": 2.5e-5, + "max_output_tokens": 128000, + }, + ): + first_reservation = await reserve_budget_for_request( + request_body=request_body, + route="/chat/completions", + llm_router=None, + valid_token=valid_token, + team_object=team_object, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + # The team counter must not be pinned at max_budget while the first + # request is in flight, otherwise concurrent requests false-positive. + team_counter_after_first = ( + counter_cache.in_memory_cache.get_cache( + key=f"spend:team:{team_object.team_id}" + ) + or 0.0 + ) + assert team_counter_after_first < team_object.max_budget, ( + f"Team counter sat at {team_counter_after_first} after one uncapped " + f"reservation against a {team_object.max_budget} budget — concurrent " + "requests will be falsely blocked." + ) + + # Second request — same shape — must succeed without raising. + second_reservation = await reserve_budget_for_request( + request_body=request_body, + route="/chat/completions", + llm_router=None, + valid_token=valid_token, + team_object=team_object, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + assert second_reservation is not None + + if first_reservation is not None: + await release_budget_reservation(first_reservation) + if second_reservation is not None: + await release_budget_reservation(second_reservation) From b5d3a5fc856ed1cf9b101d37bd0ec6d6d44751b2 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Fri, 8 May 2026 21:05:50 -0700 Subject: [PATCH 14/23] feat: add read-replica routing for Prisma DB via DATABASE_URL_READ_REPLICA (#27493) - Introduce RoutingPrismaWrapper that transparently routes read operations (find_*, count, group_by, query_raw, query_first) to a reader endpoint while writes remain on the writer, enabling Aurora-style reader/writer endpoint splits - Add IAMEndpoint dataclass and parse_iam_endpoint_from_url() to capture static connection fields from a reader URL so only the IAM token needs to rotate, avoiding the need for separate DATABASE_HOST_READ_REPLICA/etc. env vars - Enhance PrismaWrapper with per-instance knobs (db_url_env_var, iam_endpoint, recreate_uses_datasource, log_prefix) so writer and reader wrappers are independent: the reader writes its fresh URL to DATABASE_URL_READ_REPLICA and passes datasource override to Prisma since Prisma only auto-reads DATABASE_URL - Fix deadlock in PrismaWrapper.__getattr__: when called from inside a running event loop, schedule the token refresh as a background task instead of blocking with run_coroutine_threadsafe + future.result(), which would deadlock the loop thread waiting for a coroutine that needs the loop to run - Fix botocore crash when DATABASE_PORT is unset by defaulting to "5432" in both proxy_cli.py and PrismaWrapper.get_rds_iam_token(); passing None caused botocore to embed the literal string "None" in the presigned URL - Implement graceful reader degradation: reader connect/recreate failures are non-fatal; wrapper sets _reader_unavailable=True and silently routes reads to the writer to keep the proxy serving traffic during transient reader outages - Add PrismaClient.writer_db property so the reconnect smoke-test always validates the writer engine specifically; query_raw on the routing wrapper would route to the reader and not verify the newly-recreated writer - Expose DATABASE_URL_READ_REPLICA in Helm chart (values.yaml + deployment.yaml) via both plain value and secret key reference, and document the field in docker-compose.yml - Add 887-line test suite covering routing logic, IAM token refresh paths, reader degradation scenarios, datasource override behavior, and the deadlock regression Co-authored-by: Yassin Kortam --- .../litellm-helm/templates/deployment.yaml | 10 + deploy/charts/litellm-helm/values.yaml | 20 + docker-compose.yml | 5 + litellm/proxy/db/prisma_client.py | 237 +++-- litellm/proxy/db/routing_prisma_wrapper.py | 213 +++++ litellm/proxy/proxy_cli.py | 7 +- litellm/proxy/utils.py | 121 ++- .../proxy/db/test_routing_prisma_wrapper.py | 887 ++++++++++++++++++ 8 files changed, 1423 insertions(+), 77 deletions(-) create mode 100644 litellm/proxy/db/routing_prisma_wrapper.py create mode 100644 tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 6aa1771b7bb..25f69080878 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -100,6 +100,16 @@ spec: - name: DATABASE_URL value: {{ .Values.db.url | quote }} {{- end }} + {{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }} + - name: DATABASE_URL_READ_REPLICA + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.readReplicaUrlKey }} + {{- else if .Values.db.readReplicaUrl }} + - name: DATABASE_URL_READ_REPLICA + value: {{ .Values.db.readReplicaUrl | quote }} + {{- end }} - name: PROXY_MASTER_KEY valueFrom: secretKeyRef: diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index ba4059e0840..9c7c013341b 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -252,6 +252,26 @@ db: passwordKey: password # Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint endpointKey: "" + # Optional: when set, DATABASE_URL_READ_REPLICA will be sourced from this + # secret key instead of db.readReplicaUrl. Prefer this over the plain + # value: read-replica URLs typically embed credentials, and a value + # written to db.readReplicaUrl ends up visible in the rendered pod spec + # and the Helm release secret. + readReplicaUrlKey: "" + + # Optional read-replica routing. When set, the proxy sends read-only + # queries (find_*, count, group_by, query_raw/_first) to this URL while + # writes continue to go to db.url. Useful for Aurora-style clusters with + # separate reader/writer endpoints. Leave empty to keep single-DB behavior. + # When IAM_TOKEN_DB_AUTH is enabled, the reader URL is auto-refreshed + # alongside the writer (host/port/user/db are parsed from this URL once + # at startup; only the IAM token rotates). + # + # If the URL embeds credentials, prefer db.secret.readReplicaUrlKey over + # this field — the plain value is rendered into the pod spec and the + # Helm release secret. This field is intended for credential-less URLs + # only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime). + readReplicaUrl: "" # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster. # The Stackgres Operator must already be installed within the target diff --git a/docker-compose.yml b/docker-compose.yml index 988860a7877..80e1f289aad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,11 @@ services: - "4000:4000" # Map the container port to the host, change the host port if necessary environment: DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm" + # Optional: route read-only queries (find_*, count, group_by, query_raw/_first) + # to a separate reader endpoint, e.g. an Aurora reader. Leave unset for + # single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL + # is auto-refreshed alongside the writer. + # DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm" STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI env_file: - .env # Load local .env file diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index d112e222307..af5a58802bb 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -10,13 +10,64 @@ import subprocess import time import urllib import urllib.parse +from dataclasses import dataclass from datetime import datetime, timedelta -from typing import Any, Optional, Union +from typing import Any, Dict, Optional, Union from litellm._logging import verbose_proxy_logger from litellm.secret_managers.main import str_to_bool +@dataclass(frozen=True) +class IAMEndpoint: + """Static parts of an RDS IAM-authenticated Postgres connection. + + The IAM token rotates every ~15 minutes; everything else (host, port, user, + database name, schema) stays fixed. We capture the static fields once so + refresh just regenerates the token and reassembles the URL. + """ + + host: str + port: str + user: str + name: str + schema: Optional[str] = None + + def build_url(self, token: str) -> str: + url = f"postgresql://{self.user}:{token}@{self.host}:{self.port}/{self.name}" + if self.schema: + url += f"?schema={self.schema}" + return url + + +def parse_iam_endpoint_from_url(url: str) -> IAMEndpoint: + """Parse an IAMEndpoint from a Postgres URL. + + Used so a reader URL can drive its own IAM refresh without requiring + callers to set parallel DATABASE_HOST_READ_REPLICA / etc. env vars. + """ + parsed = urllib.parse.urlparse(url) + if not parsed.hostname or not parsed.username: + raise ValueError("Cannot parse IAM endpoint from URL: missing host or username") + name = (parsed.path or "/").lstrip("/") + if not name: + raise ValueError("Cannot parse IAM endpoint from URL: missing database name") + port = str(parsed.port) if parsed.port else "5432" + schema: Optional[str] = None + if parsed.query: + qs = urllib.parse.parse_qs(parsed.query) + schema_vals = qs.get("schema") + if schema_vals: + schema = schema_vals[0] + return IAMEndpoint( + host=parsed.hostname, + port=port, + user=parsed.username, + name=name, + schema=schema, + ) + + class PrismaWrapper: """ Wrapper around Prisma client that handles RDS IAM token authentication. @@ -37,10 +88,33 @@ class PrismaWrapper: # Fallback refresh interval if token parsing fails (10 minutes) FALLBACK_REFRESH_INTERVAL_SECONDS = 600 - def __init__(self, original_prisma: Any, iam_token_db_auth: bool): + def __init__( + self, + original_prisma: Any, + iam_token_db_auth: bool, + *, + db_url_env_var: str = "DATABASE_URL", + iam_endpoint: Optional[IAMEndpoint] = None, + recreate_uses_datasource: bool = False, + log_prefix: str = "", + ): self._original_prisma = original_prisma self.iam_token_db_auth = iam_token_db_auth + # Per-connection knobs so the same wrapper can be used for the writer + # (defaults: DATABASE_URL env, IAM endpoint from DATABASE_HOST/etc., + # recreate via env reload) or for a reader (DATABASE_URL_READ_REPLICA + # env, IAM endpoint parsed from that URL, recreate via datasource + # override since Prisma only auto-reads DATABASE_URL). + self._db_url_env_var = db_url_env_var + self._iam_endpoint = iam_endpoint + self._recreate_uses_datasource = recreate_uses_datasource + # Tag every log line emitted by this wrapper instance so writer and + # reader can be told apart in interleaved output (e.g. "[writer] RDS + # IAM token refresh scheduled in 720 seconds"). Empty string (default) + # keeps backward-compatible logs for the single-DB case. + self._log_prefix = f"{log_prefix} " if log_prefix else "" + # Background token refresh task management self._token_refresh_task: Optional[asyncio.Task] = None self._reconnection_lock = asyncio.Lock() @@ -157,7 +231,7 @@ class PrismaWrapper: Returns 0 if token should be refreshed immediately. Returns FALLBACK_REFRESH_INTERVAL_SECONDS if parsing fails. """ - db_url = os.getenv("DATABASE_URL") + db_url = os.getenv(self._db_url_env_var) token = self._extract_token_from_db_url(db_url) expiration_time = self._parse_token_expiration(token) @@ -199,12 +273,30 @@ class PrismaWrapper: return datetime.utcnow() > expiration_time def get_rds_iam_token(self) -> Optional[str]: - """Generate a new RDS IAM token and update DATABASE_URL.""" - if self.iam_token_db_auth: - from litellm.proxy.auth.rds_iam_token import generate_iam_auth_token + """Generate a new RDS IAM token and update the configured DB URL env var. + When the wrapper was constructed with an explicit `iam_endpoint` + (typical for a reader wrapper whose host/port/user came from a parsed + URL), use that. Otherwise fall back to the legacy DATABASE_HOST/PORT/ + USER/NAME/SCHEMA env vars (writer behavior). + """ + if not self.iam_token_db_auth: + return None + + from litellm.proxy.auth.rds_iam_token import generate_iam_auth_token + + if self._iam_endpoint is not None: + endpoint = self._iam_endpoint + token = generate_iam_auth_token( + db_host=endpoint.host, db_port=endpoint.port, db_user=endpoint.user + ) + _db_url = endpoint.build_url(token) + else: db_host = os.getenv("DATABASE_HOST") - db_port = os.getenv("DATABASE_PORT") + # Default to the Postgres standard port; passing None to + # `generate_iam_auth_token` makes botocore embed the literal + # string "None" in the presigned URL, which then fails to parse. + db_port = os.getenv("DATABASE_PORT", "5432") db_user = os.getenv("DATABASE_USER") db_name = os.getenv("DATABASE_NAME") db_schema = os.getenv("DATABASE_SCHEMA") @@ -217,9 +309,8 @@ class PrismaWrapper: if db_schema: _db_url += f"?schema={db_schema}" - os.environ["DATABASE_URL"] = _db_url - return _db_url - return None + os.environ[self._db_url_env_var] = _db_url + return _db_url async def recreate_prisma_client( self, new_db_url: str, http_client: Optional[Any] = None @@ -231,6 +322,11 @@ class PrismaWrapper: synchronous `subprocess.Popen.wait()` that can freeze the asyncio event loop for 30-120+ seconds when the engine is stuck on TCP close, breaking `/health/liveliness` and causing Kubernetes pod restarts. + + The writer wrapper relies on Prisma re-reading `DATABASE_URL` from env; + the reader wrapper opts into `recreate_uses_datasource=True` so the + new URL is passed explicitly via `datasource={"url": ...}` (Prisma + does not auto-read alternate env vars like DATABASE_URL_READ_REPLICA). """ from prisma import Prisma # type: ignore @@ -238,10 +334,12 @@ class PrismaWrapper: if old_engine_pid > 0: await self._kill_engine_process(old_engine_pid) + kwargs: Dict[str, Any] = {} if http_client is not None: - self._original_prisma = Prisma(http=http_client) - else: - self._original_prisma = Prisma() + kwargs["http"] = http_client + if self._recreate_uses_datasource: + kwargs["datasource"] = {"url": new_db_url} + self._original_prisma = Prisma(**kwargs) await self._original_prisma.connect() @@ -265,7 +363,8 @@ class PrismaWrapper: self._token_refresh_task = asyncio.create_task(self._token_refresh_loop()) verbose_proxy_logger.info( - "Started RDS IAM token proactive refresh background task" + "%sStarted RDS IAM token proactive refresh background task", + self._log_prefix, ) async def stop_token_refresh_task(self) -> None: @@ -283,7 +382,9 @@ class PrismaWrapper: except asyncio.CancelledError: pass self._token_refresh_task = None - verbose_proxy_logger.info("Stopped RDS IAM token refresh background task") + verbose_proxy_logger.info( + "%sStopped RDS IAM token refresh background task", self._log_prefix + ) async def _token_refresh_loop(self) -> None: """ @@ -294,7 +395,7 @@ class PrismaWrapper: This is more efficient than polling, requiring only 1 wake-up per token cycle. """ verbose_proxy_logger.info( - f"RDS IAM token refresh loop started. " + f"{self._log_prefix}RDS IAM token refresh loop started. " f"Tokens will be refreshed {self.TOKEN_REFRESH_BUFFER_SECONDS}s before expiration." ) @@ -305,21 +406,25 @@ class PrismaWrapper: if sleep_seconds > 0: verbose_proxy_logger.info( - f"RDS IAM token refresh scheduled in {sleep_seconds:.0f} seconds " - f"({sleep_seconds / 60:.1f} minutes)" + f"{self._log_prefix}RDS IAM token refresh scheduled in " + f"{sleep_seconds:.0f} seconds ({sleep_seconds / 60:.1f} minutes)" ) await asyncio.sleep(sleep_seconds) # Refresh the token - verbose_proxy_logger.info("Proactively refreshing RDS IAM token...") + verbose_proxy_logger.info( + "%sProactively refreshing RDS IAM token...", self._log_prefix + ) await self._safe_refresh_token() except asyncio.CancelledError: - verbose_proxy_logger.info("RDS IAM token refresh loop cancelled") + verbose_proxy_logger.info( + "%sRDS IAM token refresh loop cancelled", self._log_prefix + ) break except Exception as e: verbose_proxy_logger.error( - f"Error in RDS IAM token refresh loop: {e}. " + f"{self._log_prefix}Error in RDS IAM token refresh loop: {e}. " f"Retrying in {self.FALLBACK_REFRESH_INTERVAL_SECONDS}s..." ) # On error, wait before retrying to avoid tight error loops @@ -341,65 +446,75 @@ class PrismaWrapper: await self.recreate_prisma_client(new_db_url) self._last_refresh_time = datetime.utcnow() verbose_proxy_logger.info( - "RDS IAM token refreshed successfully. New token valid for ~15 minutes." + "%sRDS IAM token refreshed successfully. New token valid for ~15 minutes.", + self._log_prefix, ) else: verbose_proxy_logger.error( - "Failed to generate new RDS IAM token during proactive refresh" + "%sFailed to generate new RDS IAM token during proactive refresh", + self._log_prefix, ) def __getattr__(self, name: str): """ Proxy attribute access to the underlying Prisma client. - If IAM token auth is enabled and the token is expired, this method - provides a synchronous fallback to refresh the token. However, this - should rarely be needed since the background task proactively refreshes - tokens before they expire. + If IAM token auth is enabled and the token is found expired here, the + proactive refresh task has missed its window. Behavior depends on + whether we're called from inside a running event loop: - FIXED: Now properly waits for reconnection to complete before returning, - instead of the previous fire-and-forget pattern that caused the bug. + - Inside the loop (typical: from a coroutine): schedule a refresh as a + background task and return the (stale) attribute. The caller's await + will likely fail with a connection error and be retried by upper + layers (`call_with_db_reconnect_retry`); by that time the refresh + has either completed or escalated to the proactive loop's error + path. We CANNOT block here — `run_coroutine_threadsafe(...)` + + `future.result()` from inside the same loop deadlocks the loop + (loop thread is blocked, scheduled coroutine never runs, 30s timeout). + + - No running loop (sync caller, mostly tests): run the refresh in a + fresh loop and re-fetch the attribute. """ original_attr = getattr(self._original_prisma, name) if self.iam_token_db_auth: - db_url = os.getenv("DATABASE_URL") + db_url = os.getenv(self._db_url_env_var) # Check if token is expired (should be rare if background task is running) if self.is_token_expired(db_url): - verbose_proxy_logger.warning( - "RDS IAM token expired in __getattr__ - proactive refresh may have failed. " - "Triggering synchronous fallback refresh..." - ) + try: + running_loop = asyncio.get_running_loop() + except RuntimeError: + running_loop = None - new_db_url = self.get_rds_iam_token() - if new_db_url: - loop = asyncio.get_event_loop() - - if loop.is_running(): - # FIXED: Actually wait for the reconnection to complete! - # The previous code used fire-and-forget which caused the bug. - future = asyncio.run_coroutine_threadsafe( - self.recreate_prisma_client(new_db_url), loop - ) - try: - # Wait up to 30 seconds for reconnection - future.result(timeout=30) - verbose_proxy_logger.info( - "Synchronous token refresh completed successfully" - ) - except Exception as e: - verbose_proxy_logger.error( - f"Failed to refresh token synchronously: {e}" - ) - raise - else: - asyncio.run(self.recreate_prisma_client(new_db_url)) - - # Get the NEW attribute after reconnection - original_attr = getattr(self._original_prisma, name) + if running_loop is not None: + verbose_proxy_logger.warning( + "%sRDS IAM token expired in __getattr__ — proactive refresh " + "may have failed. Scheduling async refresh; the current " + "request may fail and be retried with the fresh token.", + self._log_prefix, + ) + # Non-blocking: schedule the locked refresh on the + # running loop. The reconnection lock inside + # `_safe_refresh_token` coalesces concurrent triggers. + running_loop.create_task(self._safe_refresh_token()) else: - raise ValueError("Failed to get RDS IAM token") + verbose_proxy_logger.warning( + "%sRDS IAM token expired in __getattr__ — proactive refresh " + "may have failed. Triggering synchronous fallback refresh...", + self._log_prefix, + ) + new_db_url = self.get_rds_iam_token() + if new_db_url: + asyncio.run(self.recreate_prisma_client(new_db_url)) + # Re-fetch attribute against the recreated Prisma instance. + original_attr = getattr(self._original_prisma, name) + verbose_proxy_logger.info( + "%sSynchronous token refresh completed successfully", + self._log_prefix, + ) + else: + raise ValueError("Failed to get RDS IAM token") return original_attr diff --git a/litellm/proxy/db/routing_prisma_wrapper.py b/litellm/proxy/db/routing_prisma_wrapper.py new file mode 100644 index 00000000000..0a976e9f1ea --- /dev/null +++ b/litellm/proxy/db/routing_prisma_wrapper.py @@ -0,0 +1,213 @@ +""" +RoutingPrismaWrapper: routes Prisma reads to a read-replica client and writes +to a writer client. Used when DATABASE_URL_READ_REPLICA is configured; +otherwise PrismaClient uses the writer-only PrismaWrapper directly. +""" + +import os +from typing import Any, Callable, Optional + +from litellm._logging import verbose_proxy_logger +from litellm.proxy.db.prisma_client import PrismaWrapper + +# Per-model action methods that read from the database. These are routed to +# the read replica when one is configured. +_MODEL_READ_METHODS = frozenset( + { + "find_first", + "find_first_or_raise", + "find_many", + "find_unique", + "find_unique_or_raise", + "count", + "group_by", + "query_first", + "query_raw", + } +) + +# Top-level Prisma client methods that read from the database. +_TOP_LEVEL_READ_METHODS = frozenset({"query_first", "query_raw"}) + + +class _RoutedActions: + """Per-model accessor that sends reads to the reader and writes to the writer. + + `should_use_reader` is consulted on every read dispatch so a mid-call flip + of the routing wrapper's reader-availability flag (e.g. after the reader + fails a recreate) is observed without re-fetching the actions accessor. + """ + + __slots__ = ("_writer_actions", "_reader_actions", "_should_use_reader") + + def __init__( + self, + writer_actions: Any, + reader_actions: Any, + should_use_reader: Callable[[], bool], + ): + self._writer_actions = writer_actions + self._reader_actions = reader_actions + self._should_use_reader = should_use_reader + + def __getattr__(self, name: str) -> Any: + if name in _MODEL_READ_METHODS and self._should_use_reader(): + return getattr(self._reader_actions, name) + return getattr(self._writer_actions, name) + + +class RoutingPrismaWrapper: + """ + Routes Prisma operations between a writer and a reader Prisma client. + + Reads (find_*, count, group_by, query_raw, query_first) go to the reader; + everything else (writes, transactions, raw execute) goes to the writer. + Lifecycle methods (connect, disconnect, IAM token refresh) act on both + clients so callers do not need to know about the split. When + IAM_TOKEN_DB_AUTH is enabled, both writer and reader refresh their tokens + independently on their own ~12-minute cadence. + + Reader degradation: a reader-side failure (failed connect, failed + recreate) is non-fatal — the wrapper sets `_reader_unavailable=True`, logs + a warning, and routes subsequent reads to the writer. The next successful + `connect()` or `recreate_prisma_client()` clears the flag. This keeps the + proxy serving traffic during transient reader outages instead of failing + startup or returning errors for read-heavy endpoints. + """ + + def __init__(self, writer: PrismaWrapper, reader: PrismaWrapper): + self._writer = writer + self._reader = reader + # When True, reads fall back to the writer. Flipped on by reader + # connect/recreate failures and flipped off on the next reader recovery. + self._reader_unavailable: bool = False + + @property + def writer(self) -> PrismaWrapper: + return self._writer + + @property + def reader(self) -> PrismaWrapper: + return self._reader + + @property + def reader_unavailable(self) -> bool: + return self._reader_unavailable + + def _should_use_reader(self) -> bool: + return not self._reader_unavailable + + async def connect(self, *args: Any, **kwargs: Any) -> None: + await self._writer.connect(*args, **kwargs) + verbose_proxy_logger.info("[writer] DB connected") + try: + await self._reader.connect(*args, **kwargs) + self._reader_unavailable = False + verbose_proxy_logger.info("[reader] DB connected") + except Exception as e: + # Degrade gracefully: the proxy keeps serving traffic with reads + # routed to the writer until the reader endpoint is reachable. + # Aborting startup here would tie proxy availability to an + # opt-in, best-effort reader endpoint. + self._reader_unavailable = True + verbose_proxy_logger.warning( + "Failed to connect to read replica DB: %s. " + "Falling back to the writer for reads until the reader is reachable.", + e, + ) + + async def disconnect(self, *args: Any, **kwargs: Any) -> None: + first_error: Optional[BaseException] = None + for client in (self._writer, self._reader): + try: + await client.disconnect(*args, **kwargs) + except Exception as e: + if first_error is None: + first_error = e + verbose_proxy_logger.warning("Error disconnecting Prisma client: %s", e) + if first_error is not None: + raise first_error + + def is_connected(self) -> bool: + # Reflects writer health only. The reader is best-effort; its + # availability is tracked via `_reader_unavailable` and a degraded + # reader must NOT cause a writer reconnect (would loop indefinitely + # since recreate_prisma_client only fixes writer-side problems). + return bool(self._writer.is_connected()) + + async def start_token_refresh_task(self) -> None: + await self._writer.start_token_refresh_task() + await self._reader.start_token_refresh_task() + + async def stop_token_refresh_task(self) -> None: + await self._writer.stop_token_refresh_task() + await self._reader.stop_token_refresh_task() + + async def recreate_prisma_client( + self, new_db_url: str, http_client: Optional[Any] = None + ) -> None: + """Recreate both writer and reader Prisma clients. + + The writer reconnect path in PrismaClient calls + `self.db.recreate_prisma_client(...)`. Without this method, a DB-wide + connectivity event would only re-create the writer; the reader engine + would stay broken and every routed read would fail. We always recreate + the writer first (its URL is the one passed in), then best-effort + recreate the reader. A reader failure flips `_reader_unavailable=True` + so reads transparently fall through to the writer. + """ + await self._writer.recreate_prisma_client(new_db_url, http_client=http_client) + try: + await self._recreate_reader(http_client=http_client) + self._reader_unavailable = False + except Exception as e: + self._reader_unavailable = True + verbose_proxy_logger.warning( + "Failed to recreate reader Prisma client: %s. " + "Reads will fall back to the writer until the reader recovers.", + e, + ) + + async def _recreate_reader(self, http_client: Optional[Any] = None) -> None: + """Resolve the reader URL and recreate its Prisma client. + + IAM-enabled readers regenerate their token (host/port/user came from + the parsed reader URL at construction time). Non-IAM readers reuse + the URL stored in `DATABASE_URL_READ_REPLICA`. + """ + if self._reader.iam_token_db_auth: + new_reader_url = self._reader.get_rds_iam_token() + if not new_reader_url: + raise RuntimeError( + "Failed to generate fresh IAM token for read replica" + ) + await self._reader.recreate_prisma_client( + new_reader_url, http_client=http_client + ) + return + reader_url = os.getenv("DATABASE_URL_READ_REPLICA", "") + if not reader_url: + raise RuntimeError( + "DATABASE_URL_READ_REPLICA not set; cannot recreate read replica client" + ) + await self._reader.recreate_prisma_client(reader_url, http_client=http_client) + + def __getattr__(self, name: str) -> Any: + if name in _TOP_LEVEL_READ_METHODS: + target = self._writer if self._reader_unavailable else self._reader + return getattr(target, name) + writer_attr = getattr(self._writer, name) + # Per-model action accessors are non-callable instances that expose + # both `find_many` and `create`. Methods like execute_raw / batch_ / + # tx are callables and stay on the writer untouched. + if ( + not callable(writer_attr) + and hasattr(writer_attr, "find_many") + and hasattr(writer_attr, "create") + ): + try: + reader_attr = getattr(self._reader, name) + except AttributeError: + return writer_attr + return _RoutedActions(writer_attr, reader_attr, self._should_use_reader) + return writer_attr diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 06fc0819a76..6359b48654b 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -813,7 +813,12 @@ def run_server( # noqa: PLR0915 from litellm.proxy.auth.rds_iam_token import generate_iam_auth_token db_host = os.getenv("DATABASE_HOST") - db_port = os.getenv("DATABASE_PORT") + # Default to the Postgres standard port. Without a default, + # `db_port=None` flows into `boto.generate_db_auth_token(Port=None)` + # and botocore stringifies it to `"None"` while building the + # presigned URL, which then blows up with `ValueError: Port could + # not be cast to integer value as 'None'` during signing. + db_port = os.getenv("DATABASE_PORT", "5432") db_user = os.getenv("DATABASE_USER") db_name = os.getenv("DATABASE_NAME") db_schema = os.getenv("DATABASE_SCHEMA") diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index a52dc8e55fb..0577110a26f 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -113,7 +113,11 @@ from litellm.proxy.db.exception_handler import ( call_with_db_reconnect_retry, ) from litellm.proxy.db.log_db_metrics import log_db_metrics -from litellm.proxy.db.prisma_client import PrismaWrapper +from litellm.proxy.db.prisma_client import ( + PrismaWrapper, + parse_iam_endpoint_from_url, +) +from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( UnifiedLLMGuardrails, ) @@ -2569,24 +2573,101 @@ class PrismaClient: raise Exception( "Unable to find Prisma binaries. Please run 'prisma generate' first." ) + iam_flag = ( + self.iam_token_db_auth if self.iam_token_db_auth is not None else False + ) + # When read-replica routing is on, tag log lines with [writer]/[reader] + # so the two wrappers' interleaved IAM refresh logs can be told apart. + # Single-DB deployments get an empty prefix (logs unchanged). + read_replica_url = os.getenv("DATABASE_URL_READ_REPLICA") + writer_log_prefix = "[writer]" if read_replica_url else "" if http_client is not None: - self.db = PrismaWrapper( + writer_wrapper = PrismaWrapper( original_prisma=Prisma(http=http_client), - iam_token_db_auth=( - self.iam_token_db_auth - if self.iam_token_db_auth is not None - else False - ), + iam_token_db_auth=iam_flag, + log_prefix=writer_log_prefix, ) else: - self.db = PrismaWrapper( + writer_wrapper = PrismaWrapper( original_prisma=Prisma(), - iam_token_db_auth=( - self.iam_token_db_auth - if self.iam_token_db_auth is not None - else False - ), - ) # Client to connect to Prisma db + iam_token_db_auth=iam_flag, + log_prefix=writer_log_prefix, + ) + + # Optional read-replica routing. When DATABASE_URL_READ_REPLICA is set, + # reads (find_*, count, group_by, query_raw/_first) are routed to the + # reader endpoint and writes stay on the writer. Falls back to the + # writer-only wrapper when the env var is unset, preserving existing + # single-DB deployments. + self.db: Union[PrismaWrapper, RoutingPrismaWrapper] + if read_replica_url: + try: + # If IAM auth is enabled, the reader refreshes its own token on + # the same cadence as the writer. We parse the static endpoint + # pieces (host/port/user/db) once from the reader URL — only + # the IAM token rotates after that. + reader_iam_endpoint = ( + parse_iam_endpoint_from_url(read_replica_url) if iam_flag else None + ) + # Mint a fresh IAM token for the reader BEFORE constructing the + # Prisma client. Mirrors what `proxy_cli.py` already does for + # the writer (proxy_cli.py:812-832) — without this, the reader + # Prisma is built with whatever placeholder URL the user + # supplied (no real token), and the first query falls through + # to the synchronous fallback path in + # `PrismaWrapper.__getattr__`, which deadlocks the event loop + # and times out after 30s. + if iam_flag and reader_iam_endpoint is not None: + from litellm.proxy.auth.rds_iam_token import ( + generate_iam_auth_token, + ) + + reader_token = generate_iam_auth_token( + db_host=reader_iam_endpoint.host, + db_port=reader_iam_endpoint.port, + db_user=reader_iam_endpoint.user, + ) + read_replica_url = reader_iam_endpoint.build_url(reader_token) + os.environ["DATABASE_URL_READ_REPLICA"] = read_replica_url + reader_kwargs: Dict[str, Any] = { + "datasource": {"url": read_replica_url} + } + if http_client is not None: + reader_prisma = Prisma(http=http_client, **reader_kwargs) + else: + reader_prisma = Prisma(**reader_kwargs) + reader_wrapper = PrismaWrapper( + original_prisma=reader_prisma, + iam_token_db_auth=iam_flag, + db_url_env_var="DATABASE_URL_READ_REPLICA", + iam_endpoint=reader_iam_endpoint, + recreate_uses_datasource=True, + log_prefix="[reader]", + ) + self.db = RoutingPrismaWrapper( + writer=writer_wrapper, reader=reader_wrapper + ) + verbose_proxy_logger.info( + "PrismaClient: read-replica routing enabled via DATABASE_URL_READ_REPLICA" + + (" (with IAM token auto-refresh)" if iam_flag else "") + ) + except Exception as e: + # Reader is opt-in; never let its construction fail proxy + # startup. Mirrors the runtime contract from + # `RoutingPrismaWrapper.connect`: reader-side failures are + # logged and we keep serving traffic via the writer alone. + # This recovers from transient AWS STS hiccups during the + # reader IAM token mint, malformed DATABASE_URL_READ_REPLICA, + # and Prisma construction errors. Operator restart is required + # to retry read-routing once the underlying issue is resolved. + verbose_proxy_logger.warning( + "Failed to initialize read replica Prisma client: %s. " + "Falling back to writer-only mode (no read routing) until proxy restart.", + e, + ) + self.db = writer_wrapper + else: + self.db = writer_wrapper # Client to connect to Prisma db self._db_reconnect_lock = asyncio.Lock() self._db_health_watchdog_task: Optional[asyncio.Task] = None self._db_last_reconnect_attempt_ts: float = 0.0 @@ -2624,6 +2705,13 @@ class PrismaClient: self._engine_wait_thread: Optional[threading.Thread] = None verbose_proxy_logger.debug("Success - Created Prisma Client") + @property + def writer_db(self) -> PrismaWrapper: + """Underlying writer Prisma wrapper, regardless of read-replica routing.""" + if isinstance(self.db, RoutingPrismaWrapper): + return self.db.writer + return self.db + def get_request_status( self, payload: Union[dict, SpendLogsPayload] ) -> Literal["success", "failure"]: @@ -4272,7 +4360,10 @@ class PrismaClient: self._cleanup_engine_watcher() await self.db.recreate_prisma_client(db_url) await self._start_engine_watcher() - await self.db.query_raw("SELECT 1") + # Smoke-test the writer specifically; query_raw on the routing + # wrapper sends to the reader, which would not validate the + # newly-recreated writer engine. + await self.writer_db.query_raw("SELECT 1") await asyncio.wait_for(_do_direct_reconnect(), timeout=effective_timeout) diff --git a/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py b/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py new file mode 100644 index 00000000000..8c3a2b9e2d7 --- /dev/null +++ b/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py @@ -0,0 +1,887 @@ +import asyncio +import logging +import os +import sys +from typing import Any, Dict +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../../..")) + + +# NOTE: do NOT patch sys.modules["prisma"] file-wide via an autouse fixture. +# Doing so leaks across pytest-xdist test scheduling: when a worker runs a +# routing test, then later runs test_exception_handler.py, the cached MagicMock +# attribute references break `isinstance(e, prisma.errors.X)` in +# `is_database_transport_error`. The two tests below that actually need to +# stub the prisma SDK do so per-test via monkeypatch, which is properly scoped. + + +def _make_wrappers(): + from litellm.proxy.db.prisma_client import PrismaWrapper + + writer_inner = MagicMock(name="writer_prisma") + reader_inner = MagicMock(name="reader_prisma") + writer = PrismaWrapper(original_prisma=writer_inner, iam_token_db_auth=False) + reader = PrismaWrapper(original_prisma=reader_inner, iam_token_db_auth=False) + return writer, writer_inner, reader, reader_inner + + +class _FakeActions: + """Stand-in for a Prisma per-model Actions instance (non-callable, has find_many/create).""" + + def __init__(self, name: str): + self._name = name + for method in ( + "find_many", + "find_unique", + "find_first", + "count", + "group_by", + "create", + "update", + "upsert", + "delete", + "delete_many", + "update_many", + ): + setattr(self, method, MagicMock(name=f"{name}.{method}")) + + +def _model_actions_mock(name: str) -> _FakeActions: + return _FakeActions(name) + + +def test_top_level_query_raw_routes_to_reader(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + # query_raw should resolve to the reader's underlying client. + assert routing.query_raw is reader_inner.query_raw + assert routing.query_first is reader_inner.query_first + + +def test_top_level_execute_raw_routes_to_writer(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + # execute_raw, batch_, tx are write-side and must hit the writer. + assert routing.execute_raw is writer_inner.execute_raw + assert routing.batch_ is writer_inner.batch_ + assert routing.tx is writer_inner.tx + + +def test_per_model_reads_route_to_reader_writes_to_writer(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.litellm_usertable = _model_actions_mock("writer_users") + reader_inner.litellm_usertable = _model_actions_mock("reader_users") + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + actions = routing.litellm_usertable + + # Reads → reader actions. + assert actions.find_many is reader_inner.litellm_usertable.find_many + assert actions.find_unique is reader_inner.litellm_usertable.find_unique + assert actions.find_first is reader_inner.litellm_usertable.find_first + assert actions.count is reader_inner.litellm_usertable.count + assert actions.group_by is reader_inner.litellm_usertable.group_by + + # Writes → writer actions. + assert actions.create is writer_inner.litellm_usertable.create + assert actions.update is writer_inner.litellm_usertable.update + assert actions.upsert is writer_inner.litellm_usertable.upsert + assert actions.delete is writer_inner.litellm_usertable.delete + assert actions.update_many is writer_inner.litellm_usertable.update_many + assert actions.delete_many is writer_inner.litellm_usertable.delete_many + + +@pytest.mark.asyncio +async def test_connect_invokes_both_clients(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.connect = AsyncMock() + reader_inner.connect = AsyncMock() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + await routing.connect() + + writer_inner.connect.assert_awaited_once() + reader_inner.connect.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_connect_logs_writer_and_reader_success(caplog): + """Successful startup emits a positive INFO confirmation for both writer + and reader so operators can verify connectivity without inspecting the URL + in logs.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.connect = AsyncMock() + reader_inner.connect = AsyncMock() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"): + await routing.connect() + + messages = [r.getMessage() for r in caplog.records] + assert "[writer] DB connected" in messages + assert "[reader] DB connected" in messages + + +@pytest.mark.asyncio +async def test_disconnect_continues_when_one_side_fails(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.disconnect = AsyncMock(side_effect=RuntimeError("writer down")) + reader_inner.disconnect = AsyncMock() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + with pytest.raises(RuntimeError, match="writer down"): + await routing.disconnect() + + # Reader still attempted even though writer raised. + reader_inner.disconnect.assert_awaited_once() + + +def test_is_connected_reflects_writer_only(): + """is_connected() must NOT depend on reader health — a healthy writer with + a degraded reader should report True so that PrismaClient.connect()'s + health check does not re-trigger a writer reconnect (which only fixes + writer-side problems and would loop indefinitely).""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + writer_inner.is_connected = MagicMock(return_value=True) + reader_inner.is_connected = MagicMock(return_value=True) + assert routing.is_connected() is True + + # Reader down → still True (reader degradation is tracked separately). + reader_inner.is_connected = MagicMock(return_value=False) + assert routing.is_connected() is True + + # Writer down → False. + writer_inner.is_connected = MagicMock(return_value=False) + assert routing.is_connected() is False + + +def test_token_refresh_delegates_to_both_writer_and_reader(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.start_token_refresh_task = AsyncMock() + writer.stop_token_refresh_task = AsyncMock() + reader = MagicMock() + reader.start_token_refresh_task = AsyncMock() + reader.stop_token_refresh_task = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + asyncio.run(routing.start_token_refresh_task()) + asyncio.run(routing.stop_token_refresh_task()) + + # Both wrappers get start/stop — each manages its own IAM token. When + # IAM is disabled on a wrapper its task body is a no-op. + writer.start_token_refresh_task.assert_awaited_once() + writer.stop_token_refresh_task.assert_awaited_once() + reader.start_token_refresh_task.assert_awaited_once() + reader.stop_token_refresh_task.assert_awaited_once() + + +def test_routed_actions_falls_back_to_writer_for_unknown_methods(): + from litellm.proxy.db.routing_prisma_wrapper import _RoutedActions + + writer_actions = _model_actions_mock("writer") + writer_actions.some_custom_method = "writer-custom" + reader_actions = _model_actions_mock("reader") + reader_actions.some_custom_method = "reader-custom" + + routed = _RoutedActions(writer_actions, reader_actions, lambda: True) + # Unknown method → defaults to writer (safe fallback for write-like ops). + assert routed.some_custom_method == "writer-custom" + + +def test_routed_actions_respects_should_use_reader_flag(): + """When the routing wrapper marks the reader unavailable, _RoutedActions + must redirect reads to the writer instead — without needing to re-fetch + the actions accessor.""" + from litellm.proxy.db.routing_prisma_wrapper import _RoutedActions + + writer_actions = _model_actions_mock("writer") + reader_actions = _model_actions_mock("reader") + + use_reader = {"value": True} + routed = _RoutedActions(writer_actions, reader_actions, lambda: use_reader["value"]) + + # Reader healthy → reads to reader. + assert routed.find_many is reader_actions.find_many + + # Reader degrades mid-flight → next read goes to writer. + use_reader["value"] = False + assert routed.find_many is writer_actions.find_many + + +# --------------------------------------------------------------------------- +# Reader graceful degradation +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_connect_swallows_reader_failure_and_falls_back_to_writer(): + """A reader connect failure must NOT abort proxy startup. The wrapper + flips into degraded mode so subsequent reads route to the writer.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.connect = AsyncMock() + reader_inner.connect = AsyncMock(side_effect=RuntimeError("reader unreachable")) + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + # Must not raise — reader failure is non-fatal. + await routing.connect() + + assert routing.reader_unavailable is True + writer_inner.connect.assert_awaited_once() + reader_inner.connect.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_reads_route_to_writer_when_reader_unavailable(): + """Top-level read methods and per-model reads must fall through to the + writer while the reader is degraded.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.litellm_usertable = _model_actions_mock("writer_users") + reader_inner.litellm_usertable = _model_actions_mock("reader_users") + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + routing._reader_unavailable = True + + # Top-level reads → writer. + assert routing.query_raw is writer_inner.query_raw + assert routing.query_first is writer_inner.query_first + + # Per-model reads → writer actions. + actions = routing.litellm_usertable + assert actions.find_many is writer_inner.litellm_usertable.find_many + assert actions.find_unique is writer_inner.litellm_usertable.find_unique + + +@pytest.mark.asyncio +async def test_recreate_prisma_client_recreates_both_writer_and_reader(): + """Writer reconnect path calls recreate_prisma_client. The routing wrapper + must recreate BOTH clients so a DB-wide event doesn't leave a stale reader.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = False + reader.recreate_prisma_client = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + with patch.dict(os.environ, {"DATABASE_URL_READ_REPLICA": "reader-url"}): + await routing.recreate_prisma_client("writer-url", http_client=None) + + writer.recreate_prisma_client.assert_awaited_once_with( + "writer-url", http_client=None + ) + reader.recreate_prisma_client.assert_awaited_once_with( + "reader-url", http_client=None + ) + assert routing.reader_unavailable is False + + +@pytest.mark.asyncio +async def test_recreate_recovers_reader_after_prior_degradation(): + """If a previous connect/recreate degraded the reader, a successful + recreate must clear the flag so reads start hitting the reader again.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = False + reader.recreate_prisma_client = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + routing._reader_unavailable = True + + with patch.dict(os.environ, {"DATABASE_URL_READ_REPLICA": "reader-url"}): + await routing.recreate_prisma_client("writer-url") + + assert routing.reader_unavailable is False + + +@pytest.mark.asyncio +async def test_recreate_degrades_reader_if_reader_recreate_fails(): + """If the reader recreate fails, writer recreate still succeeds and the + routing wrapper degrades (does not raise).""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = False + reader.recreate_prisma_client = AsyncMock( + side_effect=RuntimeError("reader still down") + ) + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + with patch.dict(os.environ, {"DATABASE_URL_READ_REPLICA": "reader-url"}): + # Must not raise — writer was recreated, reader is best-effort. + await routing.recreate_prisma_client("writer-url") + + writer.recreate_prisma_client.assert_awaited_once() + assert routing.reader_unavailable is True + + +@pytest.mark.asyncio +async def test_recreate_degrades_reader_when_replica_url_missing(): + """Non-IAM reader needs DATABASE_URL_READ_REPLICA. If it's missing + (configuration drift), the wrapper degrades instead of raising.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = False + reader.recreate_prisma_client = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + # Ensure env var is absent. + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("DATABASE_URL_READ_REPLICA", None) + await routing.recreate_prisma_client("writer-url") + + writer.recreate_prisma_client.assert_awaited_once() + reader.recreate_prisma_client.assert_not_awaited() + assert routing.reader_unavailable is True + + +@pytest.mark.asyncio +async def test_recreate_iam_reader_refreshes_token(): + """IAM-enabled readers must refresh their token (reader has its own parsed + endpoint) and pass the fresh URL to recreate_prisma_client.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = True + reader.get_rds_iam_token = MagicMock(return_value="postgresql://u:fresh@h:5432/db") + reader.recreate_prisma_client = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + await routing.recreate_prisma_client("writer-url") + + reader.get_rds_iam_token.assert_called_once() + reader.recreate_prisma_client.assert_awaited_once_with( + "postgresql://u:fresh@h:5432/db", http_client=None + ) + assert routing.reader_unavailable is False + + +@pytest.mark.asyncio +async def test_recreate_degrades_when_iam_token_generation_returns_none(): + """If `get_rds_iam_token` returns None (e.g. AWS-side failure), the wrapper + must degrade rather than crash — this exercises the explicit `raise + RuntimeError` inside `_recreate_reader`'s IAM branch.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = True + reader.get_rds_iam_token = MagicMock(return_value=None) + reader.recreate_prisma_client = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + await routing.recreate_prisma_client("writer-url") + + writer.recreate_prisma_client.assert_awaited_once() + reader.recreate_prisma_client.assert_not_awaited() + assert routing.reader_unavailable is True + + +def test_writer_and_reader_properties_expose_underlying_wrappers(): + """The `writer` and `reader` properties are used by PrismaClient.writer_db + to smoke-test the writer specifically during reconnect — they must return + the exact wrappers passed in.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, _, reader, _ = _make_wrappers() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + assert routing.writer is writer + assert routing.reader is reader + + +def test_per_model_accessor_falls_back_when_reader_lacks_attr(): + """If the reader Prisma client somehow lacks a model accessor that the + writer has (older client / partial mock), the wrapper must fall back to + the writer accessor instead of raising AttributeError to the caller.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + # Plain class with only the accessor set on the writer side. Using a real + # class instead of MagicMock so attribute access raises AttributeError + # naturally instead of auto-creating mock attributes. + class _PartialPrisma: + pass + + writer_inner = _PartialPrisma() + writer_inner.litellm_usertable = _model_actions_mock("writer_users") + reader_inner = _PartialPrisma() # deliberately missing litellm_usertable + + writer = PrismaWrapper(original_prisma=writer_inner, iam_token_db_auth=False) + reader = PrismaWrapper(original_prisma=reader_inner, iam_token_db_auth=False) + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + actions = routing.litellm_usertable + # Falls back to the writer's accessor verbatim — not a _RoutedActions wrapper. + assert actions is writer_inner.litellm_usertable + + +@pytest.mark.asyncio +async def test_writer_recreate_passes_http_client_through(monkeypatch): + """When PrismaClient is constructed with an http_client, recreate must + forward it to the new Prisma() so connection settings persist across + reconnects.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + captured_kwargs: Dict[str, Any] = {} + + class FakePrisma: + def __init__(self, **kwargs): + captured_kwargs.update(kwargs) + + async def connect(self): + return None + + fake_module = MagicMock() + fake_module.Prisma = FakePrisma + monkeypatch.setitem(sys.modules, "prisma", fake_module) + + writer = PrismaWrapper(original_prisma=MagicMock(), iam_token_db_auth=False) + sentinel_http = object() + await writer.recreate_prisma_client( + "postgresql://u:p@h:5432/db", http_client=sentinel_http + ) + + assert captured_kwargs == {"http": sentinel_http} + + +# --------------------------------------------------------------------------- +# IAM endpoint parsing + reader IAM refresh +# --------------------------------------------------------------------------- + + +def test_parse_iam_endpoint_from_url_extracts_all_fields(): + from litellm.proxy.db.prisma_client import parse_iam_endpoint_from_url + + ep = parse_iam_endpoint_from_url( + "postgresql://litellm_user:initial-token@aurora-reader.example.com:6543/litellm?schema=public" + ) + assert ep.host == "aurora-reader.example.com" + assert ep.port == "6543" + assert ep.user == "litellm_user" + assert ep.name == "litellm" + assert ep.schema == "public" + + +def test_parse_iam_endpoint_defaults_port_to_5432_and_skips_schema(): + from litellm.proxy.db.prisma_client import parse_iam_endpoint_from_url + + ep = parse_iam_endpoint_from_url("postgresql://u@host/dbname") + assert ep.host == "host" + assert ep.port == "5432" + assert ep.user == "u" + assert ep.name == "dbname" + assert ep.schema is None + + +def test_parse_iam_endpoint_rejects_url_without_user_or_dbname(): + from litellm.proxy.db.prisma_client import parse_iam_endpoint_from_url + + with pytest.raises(ValueError, match="missing host or username"): + parse_iam_endpoint_from_url("postgresql://host:5432/db") + with pytest.raises(ValueError, match="missing database name"): + parse_iam_endpoint_from_url("postgresql://u@host:5432/") + + +def test_iam_endpoint_build_url_inserts_token_verbatim(): + from litellm.proxy.db.prisma_client import IAMEndpoint + + # `generate_iam_auth_token` already URL-encodes the presigned token, so + # `build_url` must NOT encode again — double-encoding turned `%3D` into + # `%253D` and broke RDS auth on the reader path. + ep = IAMEndpoint(host="h", port="5432", user="u", name="db", schema="public") + pre_encoded_token = "token%2Fwith%3Fweird%26chars%3Dyes" + url = ep.build_url(pre_encoded_token) + assert url == f"postgresql://u:{pre_encoded_token}@h:5432/db?schema=public" + # Sanity check: no `%25` (the encoding of `%`), confirming we didn't re-encode. + assert "%25" not in url + + +@pytest.mark.asyncio +async def test_iam_refresh_logs_carry_log_prefix(caplog): + """When `log_prefix` is set on a PrismaWrapper, every IAM-related log + line emitted by that wrapper must start with the prefix so writer and + reader can be told apart in interleaved output.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + wrapper = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + log_prefix="[reader]", + ) + + with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"): + await wrapper.start_token_refresh_task() + # Loop emits "RDS IAM token refresh loop started..." on first tick. + # Cancel immediately so the loop body runs once and we can assert. + await wrapper.stop_token_refresh_task() + + messages = [r.getMessage() for r in caplog.records] + # Both start and stop notifications carry the prefix. + assert any( + m.startswith("[reader] Started RDS IAM token proactive refresh") + for m in messages + ) + assert any( + m.startswith("[reader] Stopped RDS IAM token refresh background task") + for m in messages + ) + + +def test_get_rds_iam_token_returns_none_when_iam_disabled(): + """`get_rds_iam_token` short-circuits to None when iam_token_db_auth is + False — covers the early-return guard at the top of the method.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + wrapper = PrismaWrapper(original_prisma=MagicMock(), iam_token_db_auth=False) + assert wrapper.get_rds_iam_token() is None + + +@pytest.mark.asyncio +async def test_getattr_does_not_block_inside_running_loop_on_expired_token(monkeypatch): + """When `__getattr__` runs inside a running event loop and the IAM token + is expired, it MUST schedule the refresh as a background task and return + immediately. The previous `run_coroutine_threadsafe` + `future.result()` + pattern deadlocks the loop (loop thread blocks waiting for a coroutine + that needs the loop to run) and times out at 30s — exactly what was + breaking the reader on first query.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + # Stale URL — `is_token_expired` returns True because the password isn't + # a parseable IAM token, so we exercise the expired branch. + monkeypatch.setenv( + "DATABASE_URL_READ_REPLICA", + "postgresql://reader:placeholder@reader.aurora.local:5432/litellm", + ) + + inner = MagicMock() + inner.query_raw = MagicMock(name="query_raw_attr") + + wrapper = PrismaWrapper( + original_prisma=inner, + iam_token_db_auth=True, + db_url_env_var="DATABASE_URL_READ_REPLICA", + ) + + # Replace the heavy refresh coroutine with a no-op AsyncMock so we can + # observe whether it was scheduled without actually doing the recreate. + refresh_calls = {"count": 0} + + async def fake_refresh(): + refresh_calls["count"] += 1 + + monkeypatch.setattr(wrapper, "_safe_refresh_token", fake_refresh) + + # Direct attribute access from inside this async test runs __getattr__ + # on the loop thread, exercising the in-loop branch. If the previous + # `run_coroutine_threadsafe` + `future.result()` pattern were back, this + # line would deadlock the loop and the test would hang (and pytest's + # per-test timeout would catch it). + attr = wrapper.query_raw + # Yield once so the scheduled refresh task gets a chance to run. + await asyncio.sleep(0) + + assert attr is inner.query_raw + assert refresh_calls["count"] == 1 + + +def test_writer_get_rds_iam_token_defaults_port_when_unset(monkeypatch): + """When DATABASE_PORT is unset, the writer must default to the Postgres + standard port instead of passing `None` through. Passing None to + `generate_iam_auth_token` makes botocore embed the literal string + \"None\" in the presigned URL during signing and crashes with + `ValueError: Port could not be cast to integer value as 'None'`.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + monkeypatch.setenv("DATABASE_HOST", "writer.aurora.local") + monkeypatch.delenv("DATABASE_PORT", raising=False) + monkeypatch.setenv("DATABASE_USER", "litellm") + monkeypatch.setenv("DATABASE_NAME", "litellm") + monkeypatch.delenv("DATABASE_SCHEMA", raising=False) + monkeypatch.delenv("DATABASE_URL", raising=False) + + captured: Dict[str, Any] = {} + + def fake_generate(db_host=None, db_port=None, db_user=None): + captured["port"] = db_port + return "TOKEN" + + fake_module = MagicMock() + fake_module.generate_iam_auth_token = fake_generate + monkeypatch.setitem(sys.modules, "litellm.proxy.auth.rds_iam_token", fake_module) + + writer = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + ) + new_url = writer.get_rds_iam_token() + + assert captured["port"] == "5432" # default applied, NOT None + assert ":5432/litellm" in (new_url or "") + + +def test_writer_get_rds_iam_token_uses_database_host_env_vars(monkeypatch): + """Writer's IAM path (no iam_endpoint configured) reads host/port/user/db + from the legacy DATABASE_HOST/PORT/USER/NAME env vars and writes the URL + back to DATABASE_URL — this is the pre-read-replica behavior the patch + must preserve.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + monkeypatch.setenv("DATABASE_HOST", "writer.aurora.local") + monkeypatch.setenv("DATABASE_PORT", "5432") + monkeypatch.setenv("DATABASE_USER", "litellm") + monkeypatch.setenv("DATABASE_NAME", "litellm") + monkeypatch.setenv("DATABASE_SCHEMA", "public") + monkeypatch.delenv("DATABASE_URL", raising=False) + + captured: Dict[str, Any] = {} + + def fake_generate(db_host=None, db_port=None, db_user=None): + captured["host"] = db_host + captured["port"] = db_port + captured["user"] = db_user + return "WRITER-TOKEN" + + fake_module = MagicMock() + fake_module.generate_iam_auth_token = fake_generate + monkeypatch.setitem(sys.modules, "litellm.proxy.auth.rds_iam_token", fake_module) + + writer = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + # No iam_endpoint → legacy DATABASE_HOST/etc. path. + ) + new_url = writer.get_rds_iam_token() + + assert captured == { + "host": "writer.aurora.local", + "port": "5432", + "user": "litellm", + } + assert new_url == ( + "postgresql://litellm:WRITER-TOKEN@writer.aurora.local:5432/litellm?schema=public" + ) + # Writer updates its own env var (DATABASE_URL by default), not the reader's. + assert os.environ["DATABASE_URL"] == new_url + + +def test_reader_iam_refresh_uses_parsed_endpoint(monkeypatch): + """The reader generates fresh tokens against its parsed endpoint and + writes the new URL to DATABASE_URL_READ_REPLICA — not DATABASE_URL.""" + from litellm.proxy.db.prisma_client import IAMEndpoint, PrismaWrapper + + # Pre-seed env vars so we can prove the reader does NOT touch DATABASE_URL. + monkeypatch.setenv("DATABASE_URL", "writer-url-untouched") + monkeypatch.setenv("DATABASE_URL_READ_REPLICA", "stale-reader-url") + + captured: Dict[str, Any] = {} + + def fake_generate(db_host=None, db_port=None, db_user=None): + captured["host"] = db_host + captured["port"] = db_port + captured["user"] = db_user + return "FRESH-TOKEN" + + fake_module = MagicMock() + fake_module.generate_iam_auth_token = fake_generate + monkeypatch.setitem(sys.modules, "litellm.proxy.auth.rds_iam_token", fake_module) + + endpoint = IAMEndpoint( + host="reader.aurora.local", + port="5432", + user="lit", + name="litellm", + schema=None, + ) + reader = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + db_url_env_var="DATABASE_URL_READ_REPLICA", + iam_endpoint=endpoint, + recreate_uses_datasource=True, + ) + + new_url = reader.get_rds_iam_token() + + # IAM token generator was called with the reader's parsed endpoint, not + # the writer's DATABASE_HOST/PORT/USER env vars. + assert captured == { + "host": "reader.aurora.local", + "port": "5432", + "user": "lit", + } + assert new_url is not None + assert new_url.startswith( + "postgresql://lit:FRESH-TOKEN@reader.aurora.local:5432/litellm" + ) + # The reader updates its OWN env var; writer's DATABASE_URL is left alone. + assert os.environ["DATABASE_URL_READ_REPLICA"] == new_url + assert os.environ["DATABASE_URL"] == "writer-url-untouched" + + +@pytest.mark.asyncio +async def test_reader_recreate_uses_datasource_override(monkeypatch): + """Reader recreate must pass `datasource={"url": ...}` to Prisma() — Prisma + only auto-reads DATABASE_URL, so without the override the new reader URL + would be silently ignored.""" + from litellm.proxy.db.prisma_client import IAMEndpoint, PrismaWrapper + + captured_kwargs: Dict[str, Any] = {} + + class FakePrisma: + def __init__(self, **kwargs): + captured_kwargs.update(kwargs) + + async def connect(self): + return None + + fake_module = MagicMock() + fake_module.Prisma = FakePrisma + monkeypatch.setitem(sys.modules, "prisma", fake_module) + + reader = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + db_url_env_var="DATABASE_URL_READ_REPLICA", + iam_endpoint=IAMEndpoint(host="h", port="5432", user="u", name="db"), + recreate_uses_datasource=True, + ) + + await reader.recreate_prisma_client( + "postgresql://u:newtoken@h:5432/db", http_client=None + ) + + assert captured_kwargs == { + "datasource": {"url": "postgresql://u:newtoken@h:5432/db"} + } + + +@pytest.mark.asyncio +async def test_writer_recreate_does_not_use_datasource(monkeypatch): + """Writer keeps relying on Prisma reading DATABASE_URL from env — datasource + override must NOT leak into the writer path (would override the freshly + rotated env var).""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + captured_kwargs: Dict[str, Any] = {} + + class FakePrisma: + def __init__(self, **kwargs): + captured_kwargs.update(kwargs) + + async def connect(self): + return None + + fake_module = MagicMock() + fake_module.Prisma = FakePrisma + monkeypatch.setitem(sys.modules, "prisma", fake_module) + + writer = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + ) + + await writer.recreate_prisma_client( + "postgresql://u:newtoken@h:5432/db", http_client=None + ) + + assert "datasource" not in captured_kwargs + + +def test_prisma_client_init_falls_back_to_writer_when_reader_iam_token_fails( + monkeypatch, caplog +): + """A transient AWS STS error (or any other failure) during the reader + IAM token mint must NOT abort proxy startup. The reader is opt-in, so + `PrismaClient.__init__` should log a warning and fall back to the + writer-only `PrismaWrapper`. The runtime contract in + `RoutingPrismaWrapper.connect` already says reader-side failures are + non-fatal — but that code never runs if construction throws first.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + monkeypatch.setenv( + "DATABASE_URL_READ_REPLICA", + "postgresql://reader_user@reader.aurora.local:5432/litellm", + ) + + class FakePrisma: + def __init__(self, **kwargs): + self.kwargs = kwargs + + async def connect(self): + return None + + fake_prisma_module = MagicMock() + fake_prisma_module.Prisma = FakePrisma + monkeypatch.setitem(sys.modules, "prisma", fake_prisma_module) + + fake_iam_module = MagicMock() + + def boom(**_kwargs): + raise RuntimeError("simulated AWS STS hiccup") + + fake_iam_module.generate_iam_auth_token = boom + monkeypatch.setitem( + sys.modules, "litellm.proxy.auth.rds_iam_token", fake_iam_module + ) + + from litellm.proxy.utils import PrismaClient + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + client = PrismaClient( + database_url="postgresql://writer@writer.aurora.local:5432/litellm", + proxy_logging_obj=MagicMock(), + ) + + # Construction did not raise, and the proxy is in writer-only mode — + # NOT a RoutingPrismaWrapper, so reads will go to the writer. + assert isinstance(client.db, PrismaWrapper) + assert not isinstance(client.db, RoutingPrismaWrapper) + # And the operator gets a clear warning. + assert any( + "Failed to initialize read replica Prisma client" in r.getMessage() + for r in caplog.records + ) From 0d551ac4f06b62d270694ee877fdeccf329f5528 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 8 May 2026 21:08:55 -0700 Subject: [PATCH 15/23] fix(proxy): reserve per-image cost for image-generation requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Image-generation routes (dall-e-3, flux, etc.) have no per-token output cost so they fell through to the no-reservation read-time-only path. Concurrent image requests against a depleted budget could all pass common_checks (counter exactly at max_budget passes the strict-`>` gate) and reach the provider before reconciliation caught up. Add per-image reservation in _estimate_request_max_cost_for_model: when the model has a per-image cost field, reserve `n × cost_per_image` upfront. The atomic counter increment serializes concurrent admissions, so the second request sees the post-first-reservation counter and raises BudgetExceededError instead of silently leaking through. Both `output_cost_per_image` and `input_cost_per_image` are honored — naming is inconsistent across providers (OpenAI dall-e-3 uses input_cost_per_image, aiml/dall-e-3 uses output_cost_per_image for the same per-generated-image price). Per-pixel pricing (DALL-E 2 size variants) and TTS/STT routes still fall through to read-time enforcement; those are follow-ups. --- .../spend_tracking/budget_reservation.py | 41 +++++ .../proxy/test_budget_reservation.py | 148 ++++++++++++++++++ 2 files changed, 189 insertions(+) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index feae368d23f..1b11af84d0d 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -827,6 +827,13 @@ def _estimate_request_max_cost_for_model( if model_info is None: return None + image_cost = _estimate_image_generation_cost( + request_body=request_body, + model_info=model_info, + ) + if image_cost is not None: + return image_cost + input_cost_per_token = _to_float(model_info.get("input_cost_per_token")) output_cost_per_token = _to_float(model_info.get("output_cost_per_token")) input_tokens = _estimate_input_tokens( @@ -858,6 +865,40 @@ def _estimate_request_max_cost_for_model( return cost +def _estimate_image_generation_cost( + request_body: dict, + model_info: Dict[str, Any], +) -> Optional[float]: + """ + Reserve `n × per-image cost` for image-generation requests so concurrent + requests against a depleted budget cannot all slip past the admission gate + onto the provider. Token-based pricing (e.g. gpt-image-1) is handled by + the chat-route token path; per-pixel and size/quality-tiered pricing + (DALL-E 2 size variants, premium tiers) are not handled here and fall + through to read-time enforcement. + + The "output" vs "input" cost-per-image naming is inconsistent across + providers — OpenAI's dall-e-3 entry uses ``input_cost_per_image`` while + aiml/dall-e-3 uses ``output_cost_per_image`` — so both are summed. + """ + output_cost_per_image = _to_float(model_info.get("output_cost_per_image")) + input_cost_per_image = _to_float(model_info.get("input_cost_per_image")) + is_image_gen = ( + model_info.get("mode") == "image_generation" + or output_cost_per_image is not None + or input_cost_per_image is not None + ) + if not is_image_gen: + return None + + cost_per_image = (output_cost_per_image or 0.0) + (input_cost_per_image or 0.0) + if cost_per_image <= 0: + return None + + n = _to_int(request_body.get("n")) or 1 + return cost_per_image * max(n, 1) + + def _get_model_cost_info( model: str, llm_router: Optional[Router], diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 0e2ca98a113..751e58e7421 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -699,6 +699,154 @@ async def test_should_clamp_reservation_to_model_ceiling_when_caller_overrequest await release_budget_reservation(reservation) +@pytest.mark.asyncio +async def test_should_reserve_image_generation_cost_per_image( + spend_counter_state, +): + """Image-generation requests reserve `n × per-image cost` so concurrent + requests against a depleted budget cannot all bypass the admission gate. + The OpenAI ``dall-e-3`` entry exposes the per-image price as + ``input_cost_per_image`` (a naming quirk), while other providers use + ``output_cost_per_image`` — both must be honored.""" + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + valid_token = UserAPIKeyAuth( + token="key-image-gen", + spend=0.0, + max_budget=10.0, + ) + await key_cache.async_set_cache(key="key-image-gen", value=valid_token) + + request_body = {"model": "dall-e-3", "prompt": "a cat", "n": 3} + + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "mode": "image_generation", + "input_cost_per_image": 0.04, + }, + ): + reservation = await reserve_budget_for_request( + request_body=request_body, + route="/v1/images/generations", + llm_router=None, + valid_token=valid_token, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + assert reservation is not None + assert reservation["reserved_cost"] == pytest.approx(0.12) # 3 × $0.04 + await release_budget_reservation(reservation) + + +@pytest.mark.asyncio +async def test_should_reject_concurrent_image_request_against_depleted_budget( + spend_counter_state, +): + """Greptile P1 regression: with image-gen reservation in place, a second + concurrent image request against a budget already pinned at the cap by + the first reservation must raise BudgetExceededError instead of + silently reaching the provider.""" + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + valid_token = UserAPIKeyAuth( + token="key-image-deplete", + spend=0.0, + team_id="team-image-deplete", + ) + team_object = LiteLLM_TeamTable( + team_id="team-image-deplete", + max_budget=0.04, + spend=0.0, + ) + await key_cache.async_set_cache( + key=f"team_id:{team_object.team_id}", + value=team_object, + ) + + request_body = {"model": "dall-e-3", "prompt": "a cat"} + + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "mode": "image_generation", + "input_cost_per_image": 0.04, + }, + ): + first = await reserve_budget_for_request( + request_body=request_body, + route="/v1/images/generations", + llm_router=None, + valid_token=valid_token, + team_object=team_object, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + assert first is not None + + with pytest.raises(litellm.BudgetExceededError): + await reserve_budget_for_request( + request_body=request_body, + route="/v1/images/generations", + llm_router=None, + valid_token=valid_token, + team_object=team_object, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + await release_budget_reservation(first) + + +@pytest.mark.asyncio +async def test_should_skip_reservation_for_per_pixel_image_model( + spend_counter_state, +): + """DALL-E 2-style per-pixel pricing depends on the requested ``size``, + which we don't decode here. Fall through to read-time enforcement + rather than guess.""" + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + valid_token = UserAPIKeyAuth( + token="key-image-per-pixel", + spend=0.0, + max_budget=1.0, + ) + await key_cache.async_set_cache(key="key-image-per-pixel", value=valid_token) + + request_body = {"model": "dall-e-2", "prompt": "a cat", "size": "256x256"} + + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "mode": "image_generation", + "input_cost_per_pixel": 2.4414e-07, + "output_cost_per_pixel": 0.0, + }, + ): + reservation = await reserve_budget_for_request( + request_body=request_body, + route="/v1/images/generations", + llm_router=None, + valid_token=valid_token, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + assert reservation is None + + def test_should_start_window_without_reset_at_at_duration_boundary(): before = datetime.now(timezone.utc) - timedelta(hours=1) From 963cb4694dd39f43d01e9d02da03784dbb0b5fb7 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 9 May 2026 09:16:27 -0700 Subject: [PATCH 16/23] fix(proxy): gate image-gen reservation strictly on model mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous detection treated any model with input_cost_per_image or output_cost_per_image as image generation. Several chat and embedding models carry those fields to price multimodal vision input, not generated images: - gemini-3.1-pro-preview (mode=chat) has output_cost_per_image=0.00012 alongside input/output token pricing. - azure/gpt-realtime-* (mode=chat) has input_cost_per_image=5e-6. - amazon.titan-embed-image-v1 (mode=embedding) has input_cost_per_image=6e-5. For these models the image-gen branch fired first and reserved a fraction of a cent per request, short-circuiting the token-priced path entirely. Long Gemini chats reserved 1 × $0.00012 instead of the true token cost. Gate strictly on mode in {"image_generation", "image_edit"}. All 197 real image_generation entries and all 31 image_edit entries (Flux Kontext, Stability inpaint/outpaint, etc.) carry the right mode, so the field-presence fallback was unnecessary. Adds regression tests for the chat-model-with-image-cost-field case and for image_edit reservation. --- .../spend_tracking/budget_reservation.py | 20 ++-- .../proxy/test_budget_reservation.py | 104 ++++++++++++++++++ 2 files changed, 116 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 1b11af84d0d..200a17e2368 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -881,16 +881,20 @@ def _estimate_image_generation_cost( providers — OpenAI's dall-e-3 entry uses ``input_cost_per_image`` while aiml/dall-e-3 uses ``output_cost_per_image`` — so both are summed. """ - output_cost_per_image = _to_float(model_info.get("output_cost_per_image")) - input_cost_per_image = _to_float(model_info.get("input_cost_per_image")) - is_image_gen = ( - model_info.get("mode") == "image_generation" - or output_cost_per_image is not None - or input_cost_per_image is not None - ) - if not is_image_gen: + # Gate strictly on `mode`. Several chat and embedding models carry + # ``input_cost_per_image`` / ``output_cost_per_image`` to price multimodal + # *vision input* (e.g. ``gemini-3.1-pro-preview``, ``azure/gpt-realtime-*``, + # ``amazon.titan-embed-image-v1``). Falling back to "treat as image-gen if + # an image cost field is present" would short-circuit the token-priced + # path for those models and reserve a fraction of a cent instead of the + # true per-token cost. All real image-generation entries in + # ``model_prices_and_context_window.json`` carry ``mode: image_generation`` + # or ``mode: image_edit``, so the field-presence fallback is unnecessary. + if model_info.get("mode") not in ("image_generation", "image_edit"): return None + output_cost_per_image = _to_float(model_info.get("output_cost_per_image")) + input_cost_per_image = _to_float(model_info.get("input_cost_per_image")) cost_per_image = (output_cost_per_image or 0.0) + (input_cost_per_image or 0.0) if cost_per_image <= 0: return None diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 751e58e7421..aa0f8d63274 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -847,6 +847,110 @@ async def test_should_skip_reservation_for_per_pixel_image_model( assert reservation is None +@pytest.mark.asyncio +async def test_should_use_token_pricing_for_chat_model_with_image_cost_field( + spend_counter_state, +): + """Several chat and embedding models carry ``input_cost_per_image`` / + ``output_cost_per_image`` to price multimodal vision *input*, not image + generation (e.g. gemini-3.1-pro-preview, azure/gpt-realtime-*, + amazon.titan-embed-image-v1). _estimate_image_generation_cost must gate + on ``mode`` so these models still go through the token-priced path — + otherwise a long chat reserves a fraction of a cent instead of the true + token cost.""" + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + valid_token = UserAPIKeyAuth( + token="key-multimodal-chat", + spend=0.0, + max_budget=10.0, + ) + await key_cache.async_set_cache(key="key-multimodal-chat", value=valid_token) + + # Roughly the gemini-3.1-pro-preview shape: chat-mode model that + # carries an output_cost_per_image alongside token pricing. + output_cost_per_token = 1.2e-5 + request_body = { + "model": "gemini-3.1-pro-preview", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 1000, + } + expected_cost = 1000 * output_cost_per_token # token-priced path, not 1 × $0.00012 + + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "mode": "chat", + "input_cost_per_token": 2e-6, + "output_cost_per_token": output_cost_per_token, + "output_cost_per_image": 0.00012, + "max_output_tokens": 64000, + }, + ): + reservation = await reserve_budget_for_request( + request_body=request_body, + route="/chat/completions", + llm_router=None, + valid_token=valid_token, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + assert reservation is not None + # Token-priced path: reservation ≈ output_tokens × output_cost_per_token, + # plus a small input-token contribution. Must NOT collapse to the + # per-image price ($0.00012) which would indicate the image-gen branch + # incorrectly fired for this chat model. + assert reservation["reserved_cost"] == pytest.approx(expected_cost, rel=0.05) + assert reservation["reserved_cost"] > 0.001 # well above per-image price + await release_budget_reservation(reservation) + + +@pytest.mark.asyncio +async def test_should_reserve_image_edit_cost_per_image( + spend_counter_state, +): + """``image_edit`` models (Flux Kontext, Stability inpaint/outpaint, etc.) + bill per generated image just like ``image_generation`` and must get + the same atomic per-image reservation.""" + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + valid_token = UserAPIKeyAuth( + token="key-image-edit", + spend=0.0, + max_budget=10.0, + ) + await key_cache.async_set_cache(key="key-image-edit", value=valid_token) + + request_body = {"model": "stability/inpaint", "prompt": "a cat", "n": 2} + + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "mode": "image_edit", + "output_cost_per_image": 0.05, + }, + ): + reservation = await reserve_budget_for_request( + request_body=request_body, + route="/v1/images/edits", + llm_router=None, + valid_token=valid_token, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + assert reservation is not None + assert reservation["reserved_cost"] == pytest.approx(0.10) # 2 × $0.05 + await release_budget_reservation(reservation) + + def test_should_start_window_without_reset_at_at_duration_boundary(): before = datetime.now(timezone.utc) - timedelta(hours=1) From 0f11b3e187b9662bdc51262c7f5cbd8a141f4936 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 9 May 2026 09:36:49 -0700 Subject: [PATCH 17/23] [UI] Rename "Default" key type to "Full Access" and reorder dropdown (#27218) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Create New Key dialog had a "Default" key-type option that was misleading: it was not the actual default selection (AI APIs is), and it grants access to all routes — broader than the description implied. Rename the option to "Full Access" with an accurate description, and reorder the dropdown to AI APIs → Management → Full Access. Switch the inner labels to antd Typography for consistency with the rest of the UI. UI-only change; the underlying enum value ("default") is unchanged so the API contract is preserved. --- .../organisms/create_key_button.tsx | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 29050907503..7d3f077dafc 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -9,7 +9,7 @@ import { formatNumberWithCommas } from "@/utils/dataUtils"; import { InfoCircleOutlined } from "@ant-design/icons"; import { useQueryClient } from "@tanstack/react-query"; import { Accordion, AccordionBody, AccordionHeader, Button, Col, Grid, Text, TextInput, Title } from "@tremor/react"; -import { Button as Button2, Form, Input, Modal, Radio, Select, Switch, Tag, Tooltip } from "antd"; +import { Button as Button2, Form, Input, Modal, Radio, Select, Switch, Tag, Tooltip, Typography } from "antd"; import debounce from "lodash/debounce"; import React, { useCallback, useEffect, useState } from "react"; import { rolesWithWriteAccess } from "../../utils/roles"; @@ -979,28 +979,28 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp } }} > - + From 7057d53862c557cf41a47d664bf039d1764d14e8 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 9 May 2026 10:13:00 -0700 Subject: [PATCH 18/23] fix(ui-tests): add Typography to antd mock in create_key_button test The antd mock omits Typography, which caused all 15 tests in create_key_button.test.tsx to fail with "No 'Typography' export is defined on the 'antd' mock" after #27218 switched the key-type dropdown labels to Typography.Text / Typography.Paragraph. Add Typography (with .Text, .Paragraph, .Title subcomponents) to the mock so the dropdown renders in the test environment. --- .../components/organisms/create_key_button.test.tsx | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.test.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.test.tsx index 3ad59cb3693..bd139df4833 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.test.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.test.tsx @@ -155,6 +155,15 @@ vi.mock("antd", () => { const Button = ({ children, htmlType, ...props }: { children?: any; htmlType?: string }) => React.createElement("button", { ...props, type: htmlType ?? props.type }, children); + const Typography = ({ children, ...props }: { children?: any }) => + React.createElement("div", props, children); + Typography.Text = ({ children, ...props }: { children?: any }) => + React.createElement("span", props, children); + Typography.Paragraph = ({ children, ...props }: { children?: any }) => + React.createElement("p", props, children); + Typography.Title = ({ children, ...props }: { children?: any }) => + React.createElement("h1", props, children); + return { Button, Form, @@ -171,6 +180,7 @@ vi.mock("antd", () => { Switch, Tag, Tooltip, + Typography, }; }); From 9380940cedaecdac720524deb2a37933730a1339 Mon Sep 17 00:00:00 2001 From: milan-berri Date: Sat, 9 May 2026 22:10:54 +0300 Subject: [PATCH 19/23] fix(mcp): forward extra_headers for OpenAPI MCP tools (#27383) * fix(mcp): forward extra_headers for OpenAPI MCP tools OpenAPI-generated tools only applied static closure headers and BYOK Authorization via ContextVar. Copy MCPServer.extra_headers from the incoming MCP request into _request_extra_headers (set in server.py before local tool dispatch), merge in openapi_to_mcp_generator via a small helper. OAuth2 M2M: do not forward caller Authorization from raw_headers (same rule as _prepare_mcp_server_headers for managed MCP). Adds TestRequestExtraHeaders and clarifies mcp_server_manager registration comment. Fixes #26794 Co-authored-by: Cursor * refactor(mcp): access has_client_credentials on MCPServer directly Greptile: getattr default was redundant; property exists on MCPServer and mcp_server is non-None inside the extra_headers forwarding block. Co-authored-by: Cursor --------- Co-authored-by: Cursor Co-authored-by: Mateo Wang --- .../mcp_server/mcp_server_manager.py | 3 +- .../mcp_server/openapi_to_mcp_generator.py | 56 ++++- .../proxy/_experimental/mcp_server/server.py | 30 +++ .../test_openapi_to_mcp_generator.py | 196 ++++++++++++++++++ 4 files changed, 276 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 55d5e4409e8..6ad731e7113 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -506,7 +506,8 @@ class MCPServerManager: # Add any static headers from server config. # # Note: `extra_headers` on MCPServer is a List[str] of header names to forward - # from the client request (not available in this OpenAPI tool generation step). + # from each client MCP request; values are applied at call time via + # `_request_extra_headers` in server.py (not baked in here). # `static_headers` is a dict of concrete headers to always send. headers = ( merge_mcp_headers( diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index 718435cce6f..271517bb1e6 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -55,6 +55,13 @@ _request_auth_header: contextvars.ContextVar[Optional[str]] = contextvars.Contex "_request_auth_header", default=None ) +# Per-request extra headers forwarded from the client request. +# Populated from MCPServer.extra_headers names matched against raw request +# headers in server.py before dispatching to a local/OpenAPI tool handler. +_request_extra_headers: contextvars.ContextVar[Optional[Dict[str, str]]] = ( + contextvars.ContextVar("_request_extra_headers", default=None) +) + def _sanitize_path_parameter_value(param_value: Any, param_name: str) -> str: """Ensure path params cannot introduce directory traversal.""" @@ -297,6 +304,46 @@ def build_input_schema(operation: Dict[str, Any]) -> Dict[str, Any]: } +def _merge_openapi_tool_request_headers( + static_headers: Dict[str, str] +) -> Dict[str, str]: + """Merge static closure headers with per-request ContextVar overrides. + + Precedence (highest to lowest): + 1. ``_request_auth_header`` — BYOK override of ``Authorization`` + 2. ``static_headers`` — operator-configured headers baked into the + tool closure at registration time + 3. ``_request_extra_headers`` — per-request headers forwarded from + the MCP caller (allowlisted by ``MCPServer.extra_headers``) + + This matches the existing MCP invariant in + :func:`litellm.proxy._experimental.mcp_server.utils.merge_mcp_headers` + and the managed MCP path, where ``static_headers`` always wins over + caller-forwarded headers. Keeping the same precedence here prevents an + authenticated caller from overriding an operator-configured value + (e.g. a tenant id or upstream API key) by sending the same header name. + + Header names are compared case-insensitively so different casing cannot + bypass the precedence rules. + """ + request_extra = _request_extra_headers.get() or {} + static = static_headers or {} + + static_lower_names = {k.lower() for k in static} + effective_headers: Dict[str, str] = { + k: v for k, v in request_extra.items() if k.lower() not in static_lower_names + } + effective_headers.update(static) + + override_auth = _request_auth_header.get() + if override_auth: + for existing in [k for k in effective_headers if k.lower() == "authorization"]: + del effective_headers[existing] + effective_headers["Authorization"] = override_auth + + return effective_headers + + def create_tool_function( path: str, method: str, @@ -334,14 +381,7 @@ def create_tool_function( The function safely handles parameter names that aren't valid Python identifiers by using **kwargs instead of named parameters. """ - # Allow per-request auth override (e.g. BYOK credential set via ContextVar). - # The ContextVar holds the full Authorization header value, including the - # correct prefix (Bearer / ApiKey / Basic) formatted by the caller in - # server.py based on the server's configured auth_type. - effective_headers = dict(headers) - override_auth = _request_auth_header.get() - if override_auth: - effective_headers["Authorization"] = override_auth + effective_headers = _merge_openapi_tool_request_headers(headers) # Build URL from base_url and path url = base_url + path diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 54d9bbe6e28..276a6e8a3bb 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -158,6 +158,7 @@ if MCP_AVAILABLE: ) from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( _request_auth_header, + _request_extra_headers, ) from litellm.proxy._experimental.mcp_server.sse_transport import SseServerTransport from litellm.proxy._experimental.mcp_server.tool_registry import ( @@ -2195,11 +2196,40 @@ if MCP_AVAILABLE: auth_header_value = f"Basic {mcp_auth_header}" else: auth_header_value = f"Bearer {mcp_auth_header}" + + # Forward named client headers to OpenAPI tool upstream requests. + # MCPServer.extra_headers lists header names to copy from raw_headers. + # OAuth2 M2M: never take Authorization from the caller (matches + # _prepare_mcp_server_headers for managed MCP). + forwarded_headers: Optional[Dict[str, str]] = None + if mcp_server and mcp_server.extra_headers and raw_headers: + normalized_raw = { + str(k).lower(): v + for k, v in raw_headers.items() + if isinstance(k, str) + } + skip_caller_authorization = bool(mcp_server.has_client_credentials) + for header_name in mcp_server.extra_headers: + if not isinstance(header_name, str): + continue + if ( + skip_caller_authorization + and header_name.lower() == "authorization" + ): + continue + value = normalized_raw.get(header_name.lower()) + if value is not None: + if forwarded_headers is None: + forwarded_headers = {} + forwarded_headers[header_name] = value + _auth_token = _request_auth_header.set(auth_header_value) + _extra_token = _request_extra_headers.set(forwarded_headers) try: local_content = await _handle_local_mcp_tool(name, arguments) finally: _request_auth_header.reset(_auth_token) + _request_extra_headers.reset(_extra_token) response = CallToolResult(content=cast(Any, local_content), isError=False) # Try managed MCP server tool (pass the full prefixed name) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py index 957dea22f3c..39f3c767220 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py @@ -15,6 +15,8 @@ from unittest.mock import AsyncMock, patch import pytest from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + _request_auth_header, + _request_extra_headers, _resolve_param_list, _resolve_ref, build_input_schema, @@ -1011,3 +1013,197 @@ class TestRegisterToolsFromOpenAPI: assert re.match( r"^[a-zA-Z0-9_-]+$", name ), f"fallback tool name {name!r} not sanitized" + + +class TestRequestExtraHeaders: + """Tests for _request_extra_headers ContextVar forwarding in tool_function.""" + + @pytest.mark.asyncio + async def test_extra_headers_forwarded_to_upstream(self): + """Extra headers set via ContextVar are included in the upstream request.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + token = _request_extra_headers.set({"X-TOKEN": "secret-value"}) + try: + result = await func() + finally: + _request_extra_headers.reset(token) + + assert result == "ok" + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent.get("X-TOKEN") == "secret-value" + + @pytest.mark.asyncio + async def test_no_extra_headers_by_default(self): + """Without setting _request_extra_headers, no extra headers are injected.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + headers={"X-Static": "static-value"}, + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + result = await func() + + assert result == "ok" + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent == {"X-Static": "static-value"} + assert "X-TOKEN" not in headers_sent + + @pytest.mark.asyncio + async def test_extra_headers_merged_with_static_headers(self): + """Forwarded headers are passed through alongside non-conflicting static headers.""" + operation = {} + func = create_tool_function( + path="/data", + method="post", + operation=operation, + base_url="https://api.example.com", + headers={"X-Static": "static-value"}, + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("post", "created") + mock_client.return_value = async_client + + token = _request_extra_headers.set({"X-TOKEN": "dynamic-value"}) + try: + result = await func() + finally: + _request_extra_headers.reset(token) + + assert result == "created" + call_args = async_client.post.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent.get("X-Static") == "static-value" + assert headers_sent.get("X-TOKEN") == "dynamic-value" + + @pytest.mark.asyncio + async def test_static_headers_win_over_forwarded_on_conflict(self): + """Static (operator) headers must override forwarded (caller) headers on name conflict.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + headers={"X-Tenant": "operator-tenant"}, + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + token = _request_extra_headers.set({"X-Tenant": "caller-spoofed"}) + try: + result = await func() + finally: + _request_extra_headers.reset(token) + + assert result == "ok" + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent.get("X-Tenant") == "operator-tenant" + assert "caller-spoofed" not in headers_sent.values() + + @pytest.mark.asyncio + async def test_static_headers_win_case_insensitively(self): + """Forwarded header with different casing must not bypass the static-wins rule.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + headers={"X-Tenant": "operator-tenant"}, + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + token = _request_extra_headers.set({"x-tenant": "caller-spoofed"}) + try: + result = await func() + finally: + _request_extra_headers.reset(token) + + assert result == "ok" + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent.get("X-Tenant") == "operator-tenant" + assert "x-tenant" not in headers_sent + assert "caller-spoofed" not in headers_sent.values() + + @pytest.mark.asyncio + async def test_auth_header_still_overrides_extra_headers(self): + """_request_auth_header takes precedence for Authorization over extra headers.""" + operation = {} + func = create_tool_function( + path="/secure", + method="get", + operation=operation, + base_url="https://api.example.com", + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "secure-data") + mock_client.return_value = async_client + + extra_token = _request_extra_headers.set( + {"Authorization": "Bearer extra", "X-TOKEN": "token-value"} + ) + auth_token = _request_auth_header.set("Bearer byok-credential") + try: + result = await func() + finally: + _request_auth_header.reset(auth_token) + _request_extra_headers.reset(extra_token) + + assert result == "secure-data" + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent.get("Authorization") == "Bearer byok-credential" + assert headers_sent.get("X-TOKEN") == "token-value" + + @pytest.mark.asyncio + async def test_extra_headers_not_leaked_between_calls(self): + """After resetting the ContextVar, subsequent calls do not see the headers.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + token = _request_extra_headers.set({"X-TOKEN": "first-call"}) + _request_extra_headers.reset(token) + + await func() + + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert "X-TOKEN" not in headers_sent From 8686001b3b56463a8564a640a44eff50c595efdf Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 9 May 2026 13:50:22 -0700 Subject: [PATCH 20/23] build(packaging): raise jinja2 floor to 3.1.6 Our `uv.lock` already resolves jinja2 to 3.1.6, so Docker / CI installs get that version. The `pyproject.toml` floor was lagging at 3.1.0, which means downstream consumers using `--resolution=lowest-direct` or older constraint files can land on 3.1.0-3.1.5 instead of the version we actually test against. Aligns the declared floor with the resolved version so external installers see the same baseline our test matrix exercises. `uv lock` diff is metadata-only (no resolved-version drift). --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d194d467913..5cd83148d37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "importlib-metadata>=8.0.0,<9.0", "tokenizers>=0.21.0,<1.0", "click>=8.0.0,<9.0", - "jinja2>=3.1.0,<4.0", + "jinja2>=3.1.6,<4.0", "aiohttp>=3.10,<4.0", "pydantic>=2.10.0,<3.0.0", "jsonschema>=4.0.0,<5.0", diff --git a/uv.lock b/uv.lock index f8d78fe8794..ab9aba1e38f 100644 --- a/uv.lock +++ b/uv.lock @@ -3405,7 +3405,7 @@ requires-dist = [ { name = "gunicorn", marker = "extra == 'proxy'", specifier = "==23.0.0" }, { name = "httpx", specifier = ">=0.28.0,<1.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, - { name = "jinja2", specifier = ">=3.1.0,<4.0" }, + { name = "jinja2", specifier = ">=3.1.6,<4.0" }, { name = "jsonschema", specifier = ">=4.0.0,<5.0" }, { name = "langfuse", marker = "extra == 'proxy-runtime'", specifier = "==2.59.7" }, { name = "litellm-enterprise", marker = "extra == 'proxy'", editable = "enterprise" }, From fa5eae8bc9d07d8e6293e97ed1a4e6120fe83cc5 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 9 May 2026 13:51:34 -0700 Subject: [PATCH 21/23] chore: remove legacy deployment artifacts and litellm-js packages (#27541) - Remove litellm-js/proxy and litellm-js/spend-logs TypeScript packages that provided Cloudflare Worker proxy and Node.js spend logging services, as these are no longer maintained - Remove deprecated Docker variants (Dockerfile.alpine, Dockerfile.dev, Dockerfile.custom_ui, Dockerfile.health_check, Dockerfile.ghcr_base) that have been superseded by the primary Dockerfile - Remove legacy Kubernetes manifests (kub.yaml, service.yaml) from deploy/kubernetes in favor of the Helm chart - Remove stale index.yaml Helm chart index pinned to an old version (v1.43.18) - Remove dev_config.yaml development configuration file that contained hardcoded credentials and example endpoints - Clean up ~3,500 lines of unused code and configuration to reduce repository maintenance burden Co-authored-by: Yassin Kortam --- AGENTS.md | 21 +- CLAUDE.md | 2 +- deploy/Dockerfile.ghcr_base | 18 - deploy/kubernetes/kub.yaml | 56 - deploy/kubernetes/service.yaml | 12 - dev_config.yaml | 13 - docker/Dockerfile.alpine | 68 - docker/Dockerfile.custom_ui | 86 - docker/Dockerfile.dev | 121 -- docker/Dockerfile.health_check | 30 - index.yaml | 108 -- litellm-js/proxy/.npmrc | 5 - litellm-js/proxy/README.md | 8 - litellm-js/proxy/package-lock.json | 2054 ----------------------- litellm-js/proxy/package.json | 14 - litellm-js/proxy/src/index.ts | 59 - litellm-js/proxy/tsconfig.json | 17 - litellm-js/proxy/wrangler.toml | 18 - litellm-js/spend-logs/.npmrc | 5 - litellm-js/spend-logs/Dockerfile | 26 - litellm-js/spend-logs/README.md | 8 - litellm-js/spend-logs/package-lock.json | 597 ------- litellm-js/spend-logs/package.json | 13 - litellm-js/spend-logs/schema.prisma | 29 - litellm-js/spend-logs/src/_types.ts | 32 - litellm-js/spend-logs/src/index.ts | 84 - litellm-js/spend-logs/tsconfig.json | 13 - 27 files changed, 20 insertions(+), 3497 deletions(-) delete mode 100644 deploy/Dockerfile.ghcr_base delete mode 100644 deploy/kubernetes/kub.yaml delete mode 100644 deploy/kubernetes/service.yaml delete mode 100644 dev_config.yaml delete mode 100644 docker/Dockerfile.alpine delete mode 100644 docker/Dockerfile.custom_ui delete mode 100644 docker/Dockerfile.dev delete mode 100644 docker/Dockerfile.health_check delete mode 100644 index.yaml delete mode 100644 litellm-js/proxy/.npmrc delete mode 100644 litellm-js/proxy/README.md delete mode 100644 litellm-js/proxy/package-lock.json delete mode 100644 litellm-js/proxy/package.json delete mode 100644 litellm-js/proxy/src/index.ts delete mode 100644 litellm-js/proxy/tsconfig.json delete mode 100644 litellm-js/proxy/wrangler.toml delete mode 100644 litellm-js/spend-logs/.npmrc delete mode 100644 litellm-js/spend-logs/Dockerfile delete mode 100644 litellm-js/spend-logs/README.md delete mode 100644 litellm-js/spend-logs/package-lock.json delete mode 100644 litellm-js/spend-logs/package.json delete mode 100644 litellm-js/spend-logs/schema.prisma delete mode 100644 litellm-js/spend-logs/src/_types.ts delete mode 100644 litellm-js/spend-logs/src/index.ts delete mode 100644 litellm-js/spend-logs/tsconfig.json diff --git a/AGENTS.md b/AGENTS.md index 4bdbf26ae9d..e99bf79d783 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -241,10 +241,27 @@ When opening issues or pull requests, follow these templates: ### Running the proxy server -Start the proxy with a config file: +Create a minimal config file and start the proxy: + +```yaml +# config.yaml +model_list: + - model_name: fake-openai-endpoint + litellm_params: + model: openai/fake-model + api_key: fake-key + api_base: https://fake-api.example.com + +general_settings: + master_key: sk-1234 + +litellm_settings: + drop_params: True + telemetry: False +``` ```bash -uv run litellm --config dev_config.yaml --port 4000 +uv run litellm --config config.yaml --port 4000 ``` The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package. diff --git a/CLAUDE.md b/CLAUDE.md index 71e5af28ee7..938801df7c7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -146,7 +146,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components: - **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets. - **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields. - **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries. -- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`. +- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`. ### Setup Wizard (`litellm/setup_wizard.py`) - The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI). diff --git a/deploy/Dockerfile.ghcr_base b/deploy/Dockerfile.ghcr_base deleted file mode 100644 index 66e64e5b774..00000000000 --- a/deploy/Dockerfile.ghcr_base +++ /dev/null @@ -1,18 +0,0 @@ -# Use the provided base image -FROM ghcr.io/berriai/litellm:main-latest@sha256:7c311546c25e7bb6e8cafede9fcd3d0d622ac636b5c9418befaa32e85dfb0186 - -# Set the working directory to /app -WORKDIR /app - -# Copy the configuration file into the container at /app -COPY config.yaml . - -# Make sure your docker/entrypoint.sh is executable -# Convert Windows line endings to Unix -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh - -# Expose the necessary port -EXPOSE 4000/tcp - -# Override the CMD instruction with your desired command and arguments -CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug", "--run_gunicorn"] diff --git a/deploy/kubernetes/kub.yaml b/deploy/kubernetes/kub.yaml deleted file mode 100644 index d5ba500d8f0..00000000000 --- a/deploy/kubernetes/kub.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: litellm-deployment -spec: - replicas: 3 - selector: - matchLabels: - app: litellm - template: - metadata: - labels: - app: litellm - spec: - containers: - - name: litellm-container - image: ghcr.io/berriai/litellm:main-latest - imagePullPolicy: Always - env: - - name: AZURE_API_KEY - value: "d6f****" - - name: AZURE_API_BASE - value: "https://openai" - - name: LITELLM_MASTER_KEY - value: "sk-1234" - - name: DATABASE_URL - value: "postgresql://ishaan*********" - args: - - "--config" - - "/app/proxy_config.yaml" # Update the path to mount the config file - volumeMounts: # Define volume mount for proxy_config.yaml - - name: config-volume - mountPath: /app - readOnly: true - livenessProbe: - httpGet: - path: /health/liveliness - port: 4000 - initialDelaySeconds: 120 - periodSeconds: 15 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: /health/readiness - port: 4000 - initialDelaySeconds: 120 - periodSeconds: 15 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 10 - volumes: # Define volume to mount proxy_config.yaml - - name: config-volume - configMap: - name: litellm-config diff --git a/deploy/kubernetes/service.yaml b/deploy/kubernetes/service.yaml deleted file mode 100644 index 4751c837254..00000000000 --- a/deploy/kubernetes/service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: litellm-service -spec: - selector: - app: litellm - ports: - - protocol: TCP - port: 4000 - targetPort: 4000 - type: LoadBalancer \ No newline at end of file diff --git a/dev_config.yaml b/dev_config.yaml deleted file mode 100644 index 64e3c14703e..00000000000 --- a/dev_config.yaml +++ /dev/null @@ -1,13 +0,0 @@ -model_list: - - model_name: fake-openai-endpoint - litellm_params: - model: openai/fake-model - api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - -general_settings: - master_key: sk-1234 - -litellm_settings: - drop_params: True - telemetry: False diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine deleted file mode 100644 index 5de588cf4e4..00000000000 --- a/docker/Dockerfile.alpine +++ /dev/null @@ -1,68 +0,0 @@ -# Base image for building -ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856 - -# Runtime image -ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856 -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a - -FROM $UV_IMAGE AS uvbin - -FROM $LITELLM_BUILD_IMAGE AS builder - -WORKDIR /app - -COPY --from=uvbin /uv /usr/local/bin/uv -COPY --from=uvbin /uvx /usr/local/bin/uvx - -RUN apk add --no-cache gcc python3-dev musl-dev nodejs npm libsndfile - -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -# Copy dependency metadata first for layer caching -COPY pyproject.toml uv.lock ./ -COPY enterprise/pyproject.toml enterprise/ -COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ - -# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) -RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python3 - -# Copy full source tree -COPY . . - -# Install project and workspace packages (fast - deps already cached) -RUN uv sync --frozen --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python3 - -RUN prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -FROM $LITELLM_RUNTIME_IMAGE AS runtime - -RUN apk upgrade --no-cache && apk add --no-cache libsndfile nodejs npm - -WORKDIR /app -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -COPY --from=builder /app /app - -EXPOSE 4000/tcp - -ENTRYPOINT ["docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] diff --git a/docker/Dockerfile.custom_ui b/docker/Dockerfile.custom_ui deleted file mode 100644 index cc44893bf92..00000000000 --- a/docker/Dockerfile.custom_ui +++ /dev/null @@ -1,86 +0,0 @@ -# Use the provided base image -# NOTE: This is a dev/branch-specific tag. Update digest when the base image is rebuilt. -FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev - -# Set the working directory to /app -WORKDIR /app - -# Install Node.js and npm (adjust version as needed) -RUN apt-get update && apt-get upgrade -y \ - libxml2 \ - libexpat1 \ - openssl \ - libssl3 \ - git \ - libkrb5-3 \ - libglib2.0-0 \ - wget \ - libaom3 \ - libxslt1.1 \ - libgnutls30 \ - libc6 && \ - apt-get install -y --no-install-recommends nodejs npm && \ - npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ - GLOBAL="$(npm root -g)" && \ - find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ - done && \ - find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \ - sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \ - npm cache clean --force && \ - apt-get purge -y npm - -# Copy the UI source into the container -COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard - -# Set an environment variable for UI_BASE_PATH -# This can be overridden at build time -# set UI_BASE_PATH to "/ui" -ENV UI_BASE_PATH="/prod/ui" - -# Build the UI with the specified UI_BASE_PATH -WORKDIR /app/ui/litellm-dashboard -RUN npm ci -RUN UI_BASE_PATH=$UI_BASE_PATH npm run build - -# Create the destination directory -RUN mkdir -p /app/litellm/proxy/_experimental/out - -# Move the built files to the appropriate location -# Assuming the build output is in ./out directory -RUN rm -rf /app/litellm/proxy/_experimental/out/* && \ - mv ./out/* /app/litellm/proxy/_experimental/out/ - -# Switch back to the main app directory -WORKDIR /app - -# Make sure your docker/entrypoint.sh is executable -# Convert Windows line endings to Unix for entrypoint scripts -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh -RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -# Run as non-root user -RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser \ - && chown -R appuser:appuser /app -USER appuser - -# Expose the necessary port -EXPOSE 4000/tcp - -HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ - CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health')"] - -# Override the CMD instruction with your desired command and arguments -CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"] \ No newline at end of file diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev deleted file mode 100644 index ebc92a22d50..00000000000 --- a/docker/Dockerfile.dev +++ /dev/null @@ -1,121 +0,0 @@ -# Base image for building -ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d - -# Runtime image -ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a - -FROM $UV_IMAGE AS uvbin - -FROM $LITELLM_BUILD_IMAGE AS builder - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /usr/local/bin/uv -COPY --from=uvbin /uvx /usr/local/bin/uvx - -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc \ - g++ \ - python3-dev \ - libssl-dev \ - pkg-config \ - nodejs \ - npm \ - && rm -rf /var/lib/apt/lists/* - -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -# Copy dependency metadata first for layer caching -COPY pyproject.toml uv.lock ./ -COPY enterprise/pyproject.toml enterprise/ -COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ - -# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) -RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python - -# Copy full source tree -COPY . . - -# Build Admin UI before final sync -RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh - -# Install project and workspace packages (fast - deps already cached) -RUN uv sync --frozen --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python - -RUN prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -FROM $LITELLM_RUNTIME_IMAGE AS runtime - -USER root - -RUN apt-get update && apt-get upgrade -y \ - libxml2 \ - libexpat1 \ - openssl \ - libssl3 \ - git \ - libkrb5-3 \ - libglib2.0-0 \ - wget \ - libaom3 \ - libxslt1.1 \ - libgnutls30 \ - libc6 \ - && apt-get install -y --no-install-recommends \ - libssl3 \ - libatomic1 \ - nodejs \ - npm \ - && rm -rf /var/lib/apt/lists/* \ - && npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \ - && GLOBAL="$(npm root -g)" \ - && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ - done \ - && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \ - sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \ - && npm cache clean --force \ - && apt-get purge -y npm - -WORKDIR /app -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -COPY --from=builder /app /app - -EXPOSE 4000/tcp - -ENTRYPOINT ["docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] diff --git a/docker/Dockerfile.health_check b/docker/Dockerfile.health_check deleted file mode 100644 index a2e5cb9f71f..00000000000 --- a/docker/Dockerfile.health_check +++ /dev/null @@ -1,30 +0,0 @@ -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a -FROM $UV_IMAGE AS uvbin - -FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d - -WORKDIR /app - -# Copy the uv binary and the health check script. -COPY --from=uvbin /uv /usr/local/bin/uv -COPY pyproject.toml uv.lock /app/ -COPY scripts/health_check/health_check_client.py /app/health_check_client.py - -# Resolve and install the health-check dependencies from the project lockfile -# so the runtime image stays self-contained and reproducible. -RUN uv export --frozen --no-default-groups --only-group healthcheck --no-emit-project --no-hashes --output-file /tmp/health-check-requirements.txt \ - && uv pip install --system -r /tmp/health-check-requirements.txt \ - && rm /tmp/health-check-requirements.txt \ - && rm /app/pyproject.toml /app/uv.lock \ - && chmod +x /app/health_check_client.py - -# Run as non-root user -RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser -USER appuser - -# Health check -HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \ - CMD ["python", "/app/health_check_client.py", "--help"] - -# Set entrypoint -ENTRYPOINT ["python", "/app/health_check_client.py"] diff --git a/index.yaml b/index.yaml deleted file mode 100644 index 9b2461c36b5..00000000000 --- a/index.yaml +++ /dev/null @@ -1,108 +0,0 @@ -apiVersion: v1 -entries: - litellm-helm: - - apiVersion: v2 - appVersion: v1.43.18 - created: "2024-08-19T23:58:25.331689+08:00" - dependencies: - - condition: db.deployStandalone - name: postgresql - repository: oci://registry-1.docker.io/bitnamicharts - version: '>=13.3.0' - - condition: redis.enabled - name: redis - repository: oci://registry-1.docker.io/bitnamicharts - version: '>=18.0.0' - description: Call all LLM APIs using the OpenAI format - digest: 0411df3dc42868be8af3ad3e00cb252790e6bd7ad15f5b77f1ca5214573a8531 - name: litellm-helm - type: application - urls: - - https://berriai.github.io/litellm/litellm-helm-0.2.3.tgz - version: 0.2.3 - postgresql: - - annotations: - category: Database - images: | - - name: os-shell - image: docker.io/bitnami/os-shell:12-debian-12-r16 - - name: postgres-exporter - image: docker.io/bitnami/postgres-exporter:0.15.0-debian-12-r14 - - name: postgresql - image: docker.io/bitnami/postgresql:16.2.0-debian-12-r6 - licenses: Apache-2.0 - apiVersion: v2 - appVersion: 16.2.0 - created: "2024-08-19T23:58:25.335716+08:00" - dependencies: - - name: common - repository: oci://registry-1.docker.io/bitnamicharts - tags: - - bitnami-common - version: 2.x.x - description: PostgreSQL (Postgres) is an open source object-relational database - known for reliability and data integrity. ACID-compliant, it supports foreign - keys, joins, views, triggers and stored procedures. - digest: 3c8125526b06833df32e2f626db34aeaedb29d38f03d15349db6604027d4a167 - home: https://bitnami.com - icon: https://bitnami.com/assets/stacks/postgresql/img/postgresql-stack-220x234.png - keywords: - - postgresql - - postgres - - database - - sql - - replication - - cluster - maintainers: - - name: VMware, Inc. - url: https://github.com/bitnami/charts - name: postgresql - sources: - - https://github.com/bitnami/charts/tree/main/bitnami/postgresql - urls: - - https://berriai.github.io/litellm/charts/postgresql-14.3.1.tgz - version: 14.3.1 - redis: - - annotations: - category: Database - images: | - - name: kubectl - image: docker.io/bitnami/kubectl:1.29.2-debian-12-r3 - - name: os-shell - image: docker.io/bitnami/os-shell:12-debian-12-r16 - - name: redis - image: docker.io/bitnami/redis:7.2.4-debian-12-r9 - - name: redis-exporter - image: docker.io/bitnami/redis-exporter:1.58.0-debian-12-r4 - - name: redis-sentinel - image: docker.io/bitnami/redis-sentinel:7.2.4-debian-12-r7 - licenses: Apache-2.0 - apiVersion: v2 - appVersion: 7.2.4 - created: "2024-08-19T23:58:25.339392+08:00" - dependencies: - - name: common - repository: oci://registry-1.docker.io/bitnamicharts - tags: - - bitnami-common - version: 2.x.x - description: Redis(R) is an open source, advanced key-value store. It is often - referred to as a data structure server since keys can contain strings, hashes, - lists, sets and sorted sets. - digest: b2fa1835f673a18002ca864c54fadac3c33789b26f6c5e58e2851b0b14a8f984 - home: https://bitnami.com - icon: https://bitnami.com/assets/stacks/redis/img/redis-stack-220x234.png - keywords: - - redis - - keyvalue - - database - maintainers: - - name: VMware, Inc. - url: https://github.com/bitnami/charts - name: redis - sources: - - https://github.com/bitnami/charts/tree/main/bitnami/redis - urls: - - https://berriai.github.io/litellm/charts/redis-18.19.1.tgz - version: 18.19.1 -generated: "2024-08-19T23:58:25.322532+08:00" diff --git a/litellm-js/proxy/.npmrc b/litellm-js/proxy/.npmrc deleted file mode 100644 index 7999681cc35..00000000000 --- a/litellm-js/proxy/.npmrc +++ /dev/null @@ -1,5 +0,0 @@ -# Supply-chain hardening -# Packages needing lifecycle scripts: npm rebuild -ignore-scripts=true -# Protects local npm install only — npm ci (used in CI) ignores this -min-release-age=3 diff --git a/litellm-js/proxy/README.md b/litellm-js/proxy/README.md deleted file mode 100644 index cc58e962d8f..00000000000 --- a/litellm-js/proxy/README.md +++ /dev/null @@ -1,8 +0,0 @@ -``` -npm install -npm run dev -``` - -``` -npm run deploy -``` diff --git a/litellm-js/proxy/package-lock.json b/litellm-js/proxy/package-lock.json deleted file mode 100644 index 0d09fa1a6c4..00000000000 --- a/litellm-js/proxy/package-lock.json +++ /dev/null @@ -1,2054 +0,0 @@ -{ - "name": "proxy", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "hono": "4.12.16", - "openai": "4.29.2" - }, - "devDependencies": { - "@cloudflare/workers-types": "4.20260501.1", - "wrangler": "4.87.0" - } - }, - "node_modules/@cloudflare/kv-asset-handler": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", - "integrity": "sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==", - "dev": true, - "license": "MIT OR Apache-2.0", - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@cloudflare/unenv-preset": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.16.1.tgz", - "integrity": "sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==", - "dev": true, - "license": "MIT OR Apache-2.0", - "peerDependencies": { - "unenv": "2.0.0-rc.24", - "workerd": ">1.20260305.0 <2.0.0-0" - }, - "peerDependenciesMeta": { - "workerd": { - "optional": true - } - } - }, - "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260430.1.tgz", - "integrity": "sha512-ADohZUHf7NBvPp2PdZig2Opxx+hDkk3ve7jrTne3JRx9kDSB73zc4LzcEeEN8LKkbAcqZmvfRJfpChSlusu0lA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260430.1.tgz", - "integrity": "sha512-/DoYC/1wHs+YRZzzqSQg1/EHB4hiv1yV5U8FnmapRRIzVaPtnt+ApeOXeMrIdKidgKOI8TqQzgBU8xbIM7Cl4Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260430.1.tgz", - "integrity": "sha512-koJhBWvEVZPKCVFtMLp2iMHlYr+lFCF47wGbnlKdHVlemV0zTxJEyHI8aLlrhPLhBmOmYLp46rXw09/qJkRIhQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260430.1.tgz", - "integrity": "sha512-hMdapNAzNQZDXGGkg4Slydc3fRJP5FUZLJVVcZCW/+imhhJro9Z1rv5n/wfR+txKoSWhTYR8eOp8Pyi2bzLzlw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260430.1.tgz", - "integrity": "sha512-jS3ffixjb5USOwz4frw4WzCz0HrjVxkgyU3WiYb06N7hBAfN6eOrveAJ4QRef0+suK4V1vQFoB1oKdRBsXe9Dw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workers-types": { - "version": "4.20260501.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260501.1.tgz", - "integrity": "sha512-B/VX2w3my/sCqxKyWOX7SxUpFC1uD8Gh7I2zbI1d3zA8p7Tx03AFsnuEx8lYLmcd8yONAA93YsAZb1wAaLK83w==", - "dev": true, - "license": "MIT OR Apache-2.0" - }, - "node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", - "integrity": "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.3.tgz", - "integrity": "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz", - "integrity": "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.3.tgz", - "integrity": "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz", - "integrity": "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz", - "integrity": "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz", - "integrity": "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz", - "integrity": "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz", - "integrity": "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz", - "integrity": "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz", - "integrity": "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz", - "integrity": "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz", - "integrity": "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz", - "integrity": "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz", - "integrity": "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz", - "integrity": "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.3.tgz", - "integrity": "sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz", - "integrity": "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz", - "integrity": "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz", - "integrity": "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz", - "integrity": "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz", - "integrity": "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz", - "integrity": "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz", - "integrity": "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz", - "integrity": "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz", - "integrity": "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/colour": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", - "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/sharp-darwin-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", - "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-darwin-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", - "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", - "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", - "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", - "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", - "cpu": [ - "arm" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", - "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", - "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", - "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", - "cpu": [ - "riscv64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", - "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", - "cpu": [ - "s390x" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", - "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", - "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", - "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-linux-arm": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", - "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", - "cpu": [ - "arm" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", - "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-ppc64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", - "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-riscv64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", - "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", - "cpu": [ - "riscv64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-s390x": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", - "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", - "cpu": [ - "s390x" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", - "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", - "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", - "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-wasm32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", - "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", - "optional": true, - "dependencies": { - "@emnapi/runtime": "^1.7.0" - }, - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", - "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-ia32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", - "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", - "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", - "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - }, - "node_modules/@poppinss/colors": { - "version": "4.1.6", - "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", - "integrity": "sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==", - "dev": true, - "license": "MIT", - "dependencies": { - "kleur": "^4.1.5" - } - }, - "node_modules/@poppinss/dumper": { - "version": "0.6.5", - "resolved": "https://registry.npmjs.org/@poppinss/dumper/-/dumper-0.6.5.tgz", - "integrity": "sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@poppinss/colors": "^4.1.5", - "@sindresorhus/is": "^7.0.2", - "supports-color": "^10.0.0" - } - }, - "node_modules/@poppinss/exception": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/@poppinss/exception/-/exception-1.2.3.tgz", - "integrity": "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@sindresorhus/is": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", - "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sindresorhus/is?sponsor=1" - } - }, - "node_modules/@speed-highlight/core": { - "version": "1.2.15", - "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.15.tgz", - "integrity": "sha512-BMq1K3DsElxDWawkX6eLg9+CKJrTVGCBAWVuHXVUV2u0s2711qiChLSId6ikYPfxhdYocLNt3wWwSvDiTvFabw==", - "dev": true, - "license": "CC0-1.0" - }, - "node_modules/@types/node": { - "version": "18.19.130", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", - "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", - "license": "MIT", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/@types/node-fetch": { - "version": "2.6.13", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", - "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.4" - } - }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "license": "MIT", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "license": "MIT", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, - "node_modules/base-64": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", - "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==" - }, - "node_modules/blake3-wasm": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", - "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", - "dev": true, - "license": "MIT" - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/charenc": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", - "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", - "license": "BSD-3-Clause", - "engines": { - "node": "*" - } - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/cookie": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", - "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/crypt": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", - "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", - "license": "BSD-3-Clause", - "engines": { - "node": "*" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, - "node_modules/digest-fetch": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz", - "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==", - "license": "ISC", - "dependencies": { - "base-64": "^0.1.0", - "md5": "^2.3.0" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/error-stack-parser-es": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", - "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/antfu" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/esbuild": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", - "integrity": "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.3", - "@esbuild/android-arm": "0.27.3", - "@esbuild/android-arm64": "0.27.3", - "@esbuild/android-x64": "0.27.3", - "@esbuild/darwin-arm64": "0.27.3", - "@esbuild/darwin-x64": "0.27.3", - "@esbuild/freebsd-arm64": "0.27.3", - "@esbuild/freebsd-x64": "0.27.3", - "@esbuild/linux-arm": "0.27.3", - "@esbuild/linux-arm64": "0.27.3", - "@esbuild/linux-ia32": "0.27.3", - "@esbuild/linux-loong64": "0.27.3", - "@esbuild/linux-mips64el": "0.27.3", - "@esbuild/linux-ppc64": "0.27.3", - "@esbuild/linux-riscv64": "0.27.3", - "@esbuild/linux-s390x": "0.27.3", - "@esbuild/linux-x64": "0.27.3", - "@esbuild/netbsd-arm64": "0.27.3", - "@esbuild/netbsd-x64": "0.27.3", - "@esbuild/openbsd-arm64": "0.27.3", - "@esbuild/openbsd-x64": "0.27.3", - "@esbuild/openharmony-arm64": "0.27.3", - "@esbuild/sunos-x64": "0.27.3", - "@esbuild/win32-arm64": "0.27.3", - "@esbuild/win32-ia32": "0.27.3", - "@esbuild/win32-x64": "0.27.3" - } - }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/form-data": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", - "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", - "license": "MIT" - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "license": "MIT", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, - "node_modules/formdata-node/node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", - "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/hono": { - "version": "4.12.16", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz", - "integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==", - "license": "MIT", - "engines": { - "node": ">=16.9.0" - } - }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.0.0" - } - }, - "node_modules/is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "license": "MIT" - }, - "node_modules/kleur": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", - "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/md5": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", - "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", - "license": "BSD-3-Clause", - "dependencies": { - "charenc": "0.0.2", - "crypt": "0.0.2", - "is-buffer": "~1.1.6" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/miniflare": { - "version": "4.20260430.0", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260430.0.tgz", - "integrity": "sha512-MWvMm3Siho9Yj7lbJZidLs8hbrRvIcOrif2mnsHQZdvoKfedpea+GaN8XJxbpRcq0B2WzNI1BB1ihdnqes3/ZA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@cspotcode/source-map-support": "0.8.1", - "sharp": "^0.34.5", - "undici": "7.24.8", - "workerd": "1.20260430.1", - "ws": "8.18.0", - "youch": "4.1.0-beta.10" - }, - "bin": { - "miniflare": "bootstrap.js" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/openai": { - "version": "4.29.2", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.29.2.tgz", - "integrity": "sha512-cPkT6zjEcE4qU5OW/SoDDuXEsdOLrXlAORhzmaguj5xZSPlgKvLhi27sFWhLKj07Y6WKNWxcwIbzm512FzTBNQ==", - "license": "Apache-2.0", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "digest-fetch": "^1.3.0", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7", - "web-streams-polyfill": "^3.2.1" - }, - "bin": { - "openai": "bin/cli" - } - }, - "node_modules/path-to-regexp": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", - "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/pathe": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", - "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", - "dev": true, - "license": "MIT" - }, - "node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/sharp": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", - "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", - "dev": true, - "hasInstallScript": true, - "license": "Apache-2.0", - "dependencies": { - "@img/colour": "^1.0.0", - "detect-libc": "^2.1.2", - "semver": "^7.7.3" - }, - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.34.5", - "@img/sharp-darwin-x64": "0.34.5", - "@img/sharp-libvips-darwin-arm64": "1.2.4", - "@img/sharp-libvips-darwin-x64": "1.2.4", - "@img/sharp-libvips-linux-arm": "1.2.4", - "@img/sharp-libvips-linux-arm64": "1.2.4", - "@img/sharp-libvips-linux-ppc64": "1.2.4", - "@img/sharp-libvips-linux-riscv64": "1.2.4", - "@img/sharp-libvips-linux-s390x": "1.2.4", - "@img/sharp-libvips-linux-x64": "1.2.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", - "@img/sharp-libvips-linuxmusl-x64": "1.2.4", - "@img/sharp-linux-arm": "0.34.5", - "@img/sharp-linux-arm64": "0.34.5", - "@img/sharp-linux-ppc64": "0.34.5", - "@img/sharp-linux-riscv64": "0.34.5", - "@img/sharp-linux-s390x": "0.34.5", - "@img/sharp-linux-x64": "0.34.5", - "@img/sharp-linuxmusl-arm64": "0.34.5", - "@img/sharp-linuxmusl-x64": "0.34.5", - "@img/sharp-wasm32": "0.34.5", - "@img/sharp-win32-arm64": "0.34.5", - "@img/sharp-win32-ia32": "0.34.5", - "@img/sharp-win32-x64": "0.34.5" - } - }, - "node_modules/supports-color": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", - "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD", - "optional": true - }, - "node_modules/undici": { - "version": "7.24.8", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.24.8.tgz", - "integrity": "sha512-6KQ/+QxK49Z/p3HO6E5ZCZWNnCasyZLa5ExaVYyvPxUwKtbCPMKELJOqh7EqOle0t9cH/7d2TaaTRRa6Nhs4YQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.18.1" - } - }, - "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "license": "MIT" - }, - "node_modules/unenv": { - "version": "2.0.0-rc.24", - "resolved": "https://registry.npmjs.org/unenv/-/unenv-2.0.0-rc.24.tgz", - "integrity": "sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "pathe": "^2.0.3" - } - }, - "node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/workerd": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260430.1.tgz", - "integrity": "sha512-KEgIWyiw3Jmn+DCd/L3ePo5fmiiYb/UcwKvDWPf/nLLOiwShDFzDSsegU5NY/JcwgvO/QsLHVi2FYrbkcXNY5Q==", - "dev": true, - "hasInstallScript": true, - "license": "Apache-2.0", - "bin": { - "workerd": "bin/workerd" - }, - "engines": { - "node": ">=16" - }, - "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20260430.1", - "@cloudflare/workerd-darwin-arm64": "1.20260430.1", - "@cloudflare/workerd-linux-64": "1.20260430.1", - "@cloudflare/workerd-linux-arm64": "1.20260430.1", - "@cloudflare/workerd-windows-64": "1.20260430.1" - } - }, - "node_modules/wrangler": { - "version": "4.87.0", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.87.0.tgz", - "integrity": "sha512-lfhfKwLfQlowwgV0xhlYgE9fU3n0I30d4ccGY/rTCEm/n42Mjvlr0Ng3ZPNqlsrsKBcDR531V7dsPkgELvrk/Q==", - "dev": true, - "license": "MIT OR Apache-2.0", - "dependencies": { - "@cloudflare/kv-asset-handler": "0.5.0", - "@cloudflare/unenv-preset": "2.16.1", - "blake3-wasm": "2.1.5", - "esbuild": "0.27.3", - "miniflare": "4.20260430.0", - "path-to-regexp": "6.3.0", - "unenv": "2.0.0-rc.24", - "workerd": "1.20260430.1" - }, - "bin": { - "wrangler": "bin/wrangler.js", - "wrangler2": "bin/wrangler.js" - }, - "engines": { - "node": ">=22.0.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - }, - "peerDependencies": { - "@cloudflare/workers-types": "^4.20260430.1" - }, - "peerDependenciesMeta": { - "@cloudflare/workers-types": { - "optional": true - } - } - }, - "node_modules/ws": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", - "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/youch": { - "version": "4.1.0-beta.10", - "resolved": "https://registry.npmjs.org/youch/-/youch-4.1.0-beta.10.tgz", - "integrity": "sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@poppinss/colors": "^4.1.5", - "@poppinss/dumper": "^0.6.4", - "@speed-highlight/core": "^1.2.7", - "cookie": "^1.0.2", - "youch-core": "^0.3.3" - } - }, - "node_modules/youch-core": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/youch-core/-/youch-core-0.3.3.tgz", - "integrity": "sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@poppinss/exception": "^1.2.2", - "error-stack-parser-es": "^1.0.5" - } - } - } -} diff --git a/litellm-js/proxy/package.json b/litellm-js/proxy/package.json deleted file mode 100644 index 9fd94cd882f..00000000000 --- a/litellm-js/proxy/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "scripts": { - "dev": "wrangler dev src/index.ts", - "deploy": "wrangler deploy --minify src/index.ts" - }, - "dependencies": { - "hono": "4.12.16", - "openai": "4.29.2" - }, - "devDependencies": { - "@cloudflare/workers-types": "4.20260501.1", - "wrangler": "4.87.0" - } -} diff --git a/litellm-js/proxy/src/index.ts b/litellm-js/proxy/src/index.ts deleted file mode 100644 index dc5dc9c689e..00000000000 --- a/litellm-js/proxy/src/index.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { Hono } from 'hono' -import { Context } from 'hono'; -import { bearerAuth } from 'hono/bearer-auth' -import OpenAI from "openai"; - -const openai = new OpenAI({ - apiKey: "sk-1234", - baseURL: "https://openai-endpoint.ishaanjaffer0324.workers.dev" -}); - -async function call_proxy() { - const completion = await openai.chat.completions.create({ - messages: [{ role: "system", content: "You are a helpful assistant." }], - model: "gpt-3.5-turbo", - }); - - return completion -} - -const app = new Hono() - -// Middleware for API Key Authentication -const apiKeyAuth = async (c: Context, next: Function) => { - const apiKey = c.req.header('Authorization'); - if (!apiKey || apiKey !== 'Bearer sk-1234') { - return c.text('Unauthorized', 401); - } - await next(); -}; - - -app.use('/*', apiKeyAuth) - - -app.get('/', (c) => { - return c.text('Hello Hono!') -}) - - - - -// Handler for chat completions -const chatCompletionHandler = async (c: Context) => { - // Assuming your logic for handling chat completion goes here - // For demonstration, just returning a simple JSON response - const response = await call_proxy() - return c.json(response); -}; - -// Register the above handler for different POST routes with the apiKeyAuth middleware -app.post('/v1/chat/completions', chatCompletionHandler); -app.post('/chat/completions', chatCompletionHandler); - -// Example showing how you might handle dynamic segments within the URL -// Here, using ':model*' to capture the rest of the path as a parameter 'model' -app.post('/openai/deployments/:model*/chat/completions', chatCompletionHandler); - - -export default app diff --git a/litellm-js/proxy/tsconfig.json b/litellm-js/proxy/tsconfig.json deleted file mode 100644 index 28fcfb58246..00000000000 --- a/litellm-js/proxy/tsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "Bundler", - "strict": true, - "lib": [ - "ESNext" - ], - "types": [ - "@cloudflare/workers-types" - ], - "jsx": "react-jsx", - "jsxImportSource": "hono/jsx", - "skipLibCheck": true - }, -} \ No newline at end of file diff --git a/litellm-js/proxy/wrangler.toml b/litellm-js/proxy/wrangler.toml deleted file mode 100644 index e7c323dff97..00000000000 --- a/litellm-js/proxy/wrangler.toml +++ /dev/null @@ -1,18 +0,0 @@ -name = "my-app" -compatibility_date = "2023-12-01" - -# [vars] -# MY_VAR = "my-variable" - -# [[kv_namespaces]] -# binding = "MY_KV_NAMESPACE" -# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - -# [[r2_buckets]] -# binding = "MY_BUCKET" -# bucket_name = "my-bucket" - -# [[d1_databases]] -# binding = "DB" -# database_name = "my-database" -# database_id = "" diff --git a/litellm-js/spend-logs/.npmrc b/litellm-js/spend-logs/.npmrc deleted file mode 100644 index 7999681cc35..00000000000 --- a/litellm-js/spend-logs/.npmrc +++ /dev/null @@ -1,5 +0,0 @@ -# Supply-chain hardening -# Packages needing lifecycle scripts: npm rebuild -ignore-scripts=true -# Protects local npm install only — npm ci (used in CI) ignores this -min-release-age=3 diff --git a/litellm-js/spend-logs/Dockerfile b/litellm-js/spend-logs/Dockerfile deleted file mode 100644 index 5040dc74bf6..00000000000 --- a/litellm-js/spend-logs/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Use the specific Node.js v20.11.0 image -FROM node:20.18.1-alpine3.20 - -# Set the working directory inside the container -WORKDIR /app - -# Copy package.json and package-lock.json to the working directory -COPY ./litellm-js/spend-logs/package*.json ./ - -# Install dependencies -RUN npm ci - -# Install Prisma globally -RUN npm install -g prisma - -# Copy the rest of the application code -COPY ./litellm-js/spend-logs . - -# Generate Prisma client -RUN npx prisma generate - -# Expose the port that the Node.js server will run on -EXPOSE 3000 - -# Command to run the Node.js app with npm run dev -CMD ["npm", "run", "dev"] diff --git a/litellm-js/spend-logs/README.md b/litellm-js/spend-logs/README.md deleted file mode 100644 index e12b31db70a..00000000000 --- a/litellm-js/spend-logs/README.md +++ /dev/null @@ -1,8 +0,0 @@ -``` -npm install -npm run dev -``` - -``` -open http://localhost:3000 -``` diff --git a/litellm-js/spend-logs/package-lock.json b/litellm-js/spend-logs/package-lock.json deleted file mode 100644 index e33079766c9..00000000000 --- a/litellm-js/spend-logs/package-lock.json +++ /dev/null @@ -1,597 +0,0 @@ -{ - "name": "spend-logs", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "@hono/node-server": "1.19.13", - "hono": "4.12.16" - }, - "devDependencies": { - "@types/node": "20.19.25", - "tsx": "4.20.6" - } - }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", - "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", - "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", - "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", - "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", - "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", - "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", - "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", - "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", - "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", - "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", - "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", - "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", - "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", - "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", - "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", - "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", - "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", - "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", - "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", - "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", - "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", - "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", - "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", - "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", - "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", - "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@hono/node-server": { - "version": "1.19.13", - "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.13.tgz", - "integrity": "sha512-TsQLe4i2gvoTtrHje625ngThGBySOgSK3Xo2XRYOdqGN1teR8+I7vchQC46uLJi8OF62YTYA3AhSpumtkhsaKQ==", - "license": "MIT", - "engines": { - "node": ">=18.14.1" - }, - "peerDependencies": { - "hono": "^4" - } - }, - "node_modules/@types/node": { - "version": "20.19.25", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.25.tgz", - "integrity": "sha512-ZsJzA5thDQMSQO788d7IocwwQbI8B5OPzmqNvpf3NY/+MHDAS759Wo0gd2WQeXYt5AAAQjzcrTVC6SKCuYgoCQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "node_modules/esbuild": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", - "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.12", - "@esbuild/android-arm": "0.25.12", - "@esbuild/android-arm64": "0.25.12", - "@esbuild/android-x64": "0.25.12", - "@esbuild/darwin-arm64": "0.25.12", - "@esbuild/darwin-x64": "0.25.12", - "@esbuild/freebsd-arm64": "0.25.12", - "@esbuild/freebsd-x64": "0.25.12", - "@esbuild/linux-arm": "0.25.12", - "@esbuild/linux-arm64": "0.25.12", - "@esbuild/linux-ia32": "0.25.12", - "@esbuild/linux-loong64": "0.25.12", - "@esbuild/linux-mips64el": "0.25.12", - "@esbuild/linux-ppc64": "0.25.12", - "@esbuild/linux-riscv64": "0.25.12", - "@esbuild/linux-s390x": "0.25.12", - "@esbuild/linux-x64": "0.25.12", - "@esbuild/netbsd-arm64": "0.25.12", - "@esbuild/netbsd-x64": "0.25.12", - "@esbuild/openbsd-arm64": "0.25.12", - "@esbuild/openbsd-x64": "0.25.12", - "@esbuild/openharmony-arm64": "0.25.12", - "@esbuild/sunos-x64": "0.25.12", - "@esbuild/win32-arm64": "0.25.12", - "@esbuild/win32-ia32": "0.25.12", - "@esbuild/win32-x64": "0.25.12" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/get-tsconfig": { - "version": "4.14.0", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz", - "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==", - "dev": true, - "license": "MIT", - "dependencies": { - "resolve-pkg-maps": "^1.0.0" - }, - "funding": { - "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" - } - }, - "node_modules/hono": { - "version": "4.12.16", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz", - "integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==", - "license": "MIT", - "engines": { - "node": ">=16.9.0" - } - }, - "node_modules/resolve-pkg-maps": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", - "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" - } - }, - "node_modules/tsx": { - "version": "4.20.6", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz", - "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==", - "dev": true, - "license": "MIT", - "dependencies": { - "esbuild": "~0.25.0", - "get-tsconfig": "^4.7.5" - }, - "bin": { - "tsx": "dist/cli.mjs" - }, - "engines": { - "node": ">=18.0.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - } - }, - "node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, - "license": "MIT" - } - } -} diff --git a/litellm-js/spend-logs/package.json b/litellm-js/spend-logs/package.json deleted file mode 100644 index 5a7a95c5de1..00000000000 --- a/litellm-js/spend-logs/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "scripts": { - "dev": "tsx watch src/index.ts" - }, - "dependencies": { - "@hono/node-server": "1.19.13", - "hono": "4.12.16" - }, - "devDependencies": { - "@types/node": "20.19.25", - "tsx": "4.20.6" - } -} diff --git a/litellm-js/spend-logs/schema.prisma b/litellm-js/spend-logs/schema.prisma deleted file mode 100644 index b0403f277aa..00000000000 --- a/litellm-js/spend-logs/schema.prisma +++ /dev/null @@ -1,29 +0,0 @@ -generator client { - provider = "prisma-client-js" -} - -datasource client { - provider = "postgresql" - url = env("DATABASE_URL") -} - -model LiteLLM_SpendLogs { - request_id String @id - call_type String - api_key String @default("") - spend Float @default(0.0) - total_tokens Int @default(0) - prompt_tokens Int @default(0) - completion_tokens Int @default(0) - startTime DateTime - endTime DateTime - model String @default("") - api_base String @default("") - user String @default("") - metadata Json @default("{}") - cache_hit String @default("") - cache_key String @default("") - request_tags Json @default("[]") - team_id String? - end_user String? -} \ No newline at end of file diff --git a/litellm-js/spend-logs/src/_types.ts b/litellm-js/spend-logs/src/_types.ts deleted file mode 100644 index 6a9b499171e..00000000000 --- a/litellm-js/spend-logs/src/_types.ts +++ /dev/null @@ -1,32 +0,0 @@ -export type LiteLLM_IncrementSpend = { - key_transactions: Array, // [{"key": spend},..] - user_transactions: Array, - team_transactions: Array, - spend_logs_transactions: Array -} - -export type LiteLLM_IncrementObject = { - key: string, - spend: number -} - -export type LiteLLM_SpendLogs = { - request_id: string; // @id means it's a unique identifier - call_type: string; - api_key: string; // @default("") means it defaults to an empty string if not provided - spend: number; // Float in Prisma corresponds to number in TypeScript - total_tokens: number; // Int in Prisma corresponds to number in TypeScript - prompt_tokens: number; - completion_tokens: number; - startTime: Date; // DateTime in Prisma corresponds to Date in TypeScript - endTime: Date; - model: string; // @default("") means it defaults to an empty string if not provided - api_base: string; - user: string; - metadata: any; // Json type in Prisma is represented by any in TypeScript; could also use a more specific type if the structure of JSON is known - cache_hit: string; - cache_key: string; - request_tags: any; // Similarly, this could be an array or a more specific type depending on the expected structure - team_id?: string | null; // ? indicates it's optional and can be undefined, but could also be null if not provided - end_user?: string | null; -}; \ No newline at end of file diff --git a/litellm-js/spend-logs/src/index.ts b/litellm-js/spend-logs/src/index.ts deleted file mode 100644 index 3581d95c830..00000000000 --- a/litellm-js/spend-logs/src/index.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { serve } from '@hono/node-server' -import { Hono } from 'hono' -import { PrismaClient } from '@prisma/client' -import {LiteLLM_SpendLogs, LiteLLM_IncrementSpend, LiteLLM_IncrementObject} from './_types' - -const app = new Hono() -const prisma = new PrismaClient() -// In-memory storage for logs -let spend_logs: LiteLLM_SpendLogs[] = []; -const key_logs: LiteLLM_IncrementObject[] = []; -const user_logs: LiteLLM_IncrementObject[] = []; -const transaction_logs: LiteLLM_IncrementObject[] = []; - - -app.get('/', (c) => { - return c.text('Hello Hono!') -}) - -const MIN_LOGS = 1; // Minimum number of logs needed to initiate a flush -const FLUSH_INTERVAL = 5000; // Time in ms to wait before trying to flush again -const BATCH_SIZE = 100; // Preferred size of each batch to write to the database -const MAX_LOGS_PER_INTERVAL = 1000; // Maximum number of logs to flush in a single interval - -const flushLogsToDb = async () => { - if (spend_logs.length >= MIN_LOGS) { - // Limit the logs to process in this interval to MAX_LOGS_PER_INTERVAL or less - const logsToProcess = spend_logs.slice(0, MAX_LOGS_PER_INTERVAL); - - for (let i = 0; i < logsToProcess.length; i += BATCH_SIZE) { - // Create subarray for current batch, ensuring it doesn't exceed the BATCH_SIZE - const batch = logsToProcess.slice(i, i + BATCH_SIZE); - - // Convert datetime strings to Date objects - const batchWithDates = batch.map(entry => ({ - ...entry, - startTime: new Date(entry.startTime), - endTime: new Date(entry.endTime), - // Repeat for any other DateTime fields you may have - })); - - await prisma.liteLLM_SpendLogs.createMany({ - data: batchWithDates, - }); - - console.log(`Flushed ${batch.length} logs to the DB.`); - } - - // Remove the processed logs from spend_logs - spend_logs = spend_logs.slice(logsToProcess.length); - - console.log(`${logsToProcess.length} logs processed. Remaining in queue: ${spend_logs.length}`); - } else { - // This will ensure it doesn't falsely claim "No logs to flush." when it's merely below the MIN_LOGS threshold. - if(spend_logs.length > 0) { - console.log(`Accumulating logs. Currently at ${spend_logs.length}, waiting for at least ${MIN_LOGS}.`); - } else { - console.log("No logs to flush."); - } - } -}; - -// Setup interval for attempting to flush the logs -setInterval(flushLogsToDb, FLUSH_INTERVAL); - -// Route to receive log messages -app.post('/spend/update', async (c) => { - const incomingLogs = await c.req.json(); - - spend_logs.push(...incomingLogs); - - console.log(`Received and stored ${incomingLogs.length} logs. Total logs in memory: ${spend_logs.length}`); - - return c.json({ message: `Successfully stored ${incomingLogs.length} logs` }); -}); - - - -const port = 3000 -console.log(`Server is running on port ${port}`) - -serve({ - fetch: app.fetch, - port -}) diff --git a/litellm-js/spend-logs/tsconfig.json b/litellm-js/spend-logs/tsconfig.json deleted file mode 100644 index 028c03b6a81..00000000000 --- a/litellm-js/spend-logs/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "Bundler", - "strict": true, - "types": [ - "node" - ], - "jsx": "react-jsx", - "jsxImportSource": "hono/jsx", - } -} \ No newline at end of file From d67dfca1e111cbb63f662cc31878d8e475113baa Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Sat, 9 May 2026 14:47:48 -0700 Subject: [PATCH 22/23] Fix proxy auth status code tests (#27555) * Fix proxy auth status code tests Co-authored-by: ishaan-berri * Update user model access status expectation Co-authored-by: ishaan-berri --------- Co-authored-by: oss-agent-shin <279349115+oss-agent-shin@users.noreply.github.com> Co-authored-by: ishaan-berri --- litellm/proxy/auth/auth_checks.py | 6 +-- litellm/proxy/auth/auth_exception_handler.py | 2 +- litellm/proxy/auth/user_api_key_auth.py | 6 +-- tests/otel_tests/test_e2e_budgeting.py | 4 +- tests/otel_tests/test_e2e_model_access.py | 4 +- .../proxy/auth/test_auth_checks.py | 49 +++++++++++++++++++ .../proxy/auth/test_auth_exception_handler.py | 1 + .../proxy/auth/test_user_api_key_auth.py | 23 +++++++++ tests/test_openai_endpoints.py | 2 +- tests/test_users.py | 4 +- 10 files changed, 87 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index f6f99eb62c8..0b30999aa21 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -2849,7 +2849,7 @@ def _can_object_call_model( object_type=object_type ), param="model", - code=status.HTTP_401_UNAUTHORIZED, + code=status.HTTP_403_FORBIDDEN, ) @@ -3082,7 +3082,7 @@ async def can_user_call_model( message=f"User not allowed to access model. No default model access, only team models allowed. Tried to access {model}", type=ProxyErrorTypes.key_model_access_denied, param="model", - code=status.HTTP_401_UNAUTHORIZED, + code=status.HTTP_403_FORBIDDEN, ) return _can_object_call_model( @@ -3625,7 +3625,7 @@ async def _check_team_member_model_access( message=f"Team member not allowed to access model. User={valid_token.user_id}, Team={team_object.team_id}, Model={model}. Allowed member models = {member_allowed_models}", type=ProxyErrorTypes.team_model_access_denied, param="model", - code=status.HTTP_401_UNAUTHORIZED, + code=status.HTTP_403_FORBIDDEN, ) diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index 5ded8136ef3..431db4254eb 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -123,7 +123,7 @@ class UserAPIKeyAuthExceptionHandler: message=e.message, type=ProxyErrorTypes.budget_exceeded, param=None, - code=400, + code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS), ) if isinstance(e, HTTPException): raise ProxyException( diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 9d3c06e641f..4778549befc 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1107,7 +1107,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 raise ProxyException( message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", type=ProxyErrorTypes.expired_key, - code=400, + code=status.HTTP_401_UNAUTHORIZED, param=abbreviate_api_key(api_key=api_key), ) valid_token = update_valid_token_with_end_user_params( @@ -1432,7 +1432,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 raise ProxyException( message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", type=ProxyErrorTypes.expired_key, - code=400, + code=status.HTTP_401_UNAUTHORIZED, param=abbreviate_api_key(api_key=api_key), ) @@ -2417,7 +2417,7 @@ async def _run_post_custom_auth_checks( raise ProxyException( message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", type=ProxyErrorTypes.expired_key, - code=400, + code=status.HTTP_401_UNAUTHORIZED, param=( abbreviate_api_key(api_key=valid_token.token) if valid_token.token diff --git a/tests/otel_tests/test_e2e_budgeting.py b/tests/otel_tests/test_e2e_budgeting.py index 62fc8732ebd..f61befac4fb 100644 --- a/tests/otel_tests/test_e2e_budgeting.py +++ b/tests/otel_tests/test_e2e_budgeting.py @@ -25,8 +25,8 @@ async def make_calls_until_budget_exceeded(session, key: str, call_function, **k # Check error structure and values that should be consistent assert ( - error_dict["code"] == "400" - ), f"Expected error code 400, got: {error_dict['code']}" + error_dict["code"] == "429" + ), f"Expected error code 429, got: {error_dict['code']}" assert ( error_dict["type"] == "budget_exceeded" ), f"Expected error type budget_exceeded, got: {error_dict['type']}" diff --git a/tests/otel_tests/test_e2e_model_access.py b/tests/otel_tests/test_e2e_model_access.py index 7ea75a9d61d..87d85a19603 100644 --- a/tests/otel_tests/test_e2e_model_access.py +++ b/tests/otel_tests/test_e2e_model_access.py @@ -99,7 +99,7 @@ async def test_model_access_patterns(key_models, test_model, expect_success): # Assert error structure and values assert _error_body["type"] == "key_model_access_denied" assert _error_body["param"] == "model" - assert _error_body["code"] == "401" + assert _error_body["code"] == "403" assert "key not allowed to access model" in _error_body["message"] @@ -297,7 +297,7 @@ def _validate_model_access_exception( # Assert error structure and values assert _error_body["type"] == expected_type assert _error_body["param"] == "model" - assert _error_body["code"] == "401" + assert _error_body["code"] == "403" if expected_type == "key_model_access_denied": assert "key not allowed to access model" in _error_body["message"] elif expected_type == "team_model_access_denied": diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 8a854bcd6a8..26f04a4abcb 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -12,6 +12,7 @@ from datetime import datetime, timedelta import httpx import pytest +from fastapi import status import litellm from litellm.proxy._types import ( @@ -31,6 +32,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.auth_checks import ( ExperimentalUIJWTToken, + _can_object_call_model, _can_object_call_vector_stores, _check_end_user_budget, _check_team_member_budget, @@ -206,6 +208,52 @@ def test_get_key_object_from_ui_hash_key_invalid(): assert key_object is None +@pytest.mark.parametrize( + "object_type,expected_error_type", + [ + ("key", ProxyErrorTypes.key_model_access_denied), + ("team", ProxyErrorTypes.team_model_access_denied), + ("user", ProxyErrorTypes.user_model_access_denied), + ("org", ProxyErrorTypes.org_model_access_denied), + ("project", ProxyErrorTypes.project_model_access_denied), + ], +) +def test_can_object_call_model_denials_return_forbidden( + object_type, expected_error_type +): + with pytest.raises(ProxyException) as exc_info: + _can_object_call_model( + model="restricted-model", + llm_router=None, + models=["allowed-model"], + object_type=object_type, + ) + + assert exc_info.value.type == expected_error_type + assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN + + +@pytest.mark.asyncio +async def test_can_user_call_model_no_default_models_returns_forbidden(): + from litellm.proxy._types import SpecialModelNames + from litellm.proxy.auth.auth_checks import can_user_call_model + + user_object = LiteLLM_UserTable( + user_id="test-user", + models=[SpecialModelNames.no_default_models.value], + ) + + with pytest.raises(ProxyException) as exc_info: + await can_user_call_model( + model="restricted-model", + llm_router=None, + user_object=user_object, + ) + + assert exc_info.value.type == ProxyErrorTypes.key_model_access_denied + assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN + + @pytest.mark.asyncio async def test_get_key_object_should_reconnect_once_on_db_connection_error(): mock_prisma_client = MagicMock() @@ -1144,6 +1192,7 @@ async def test_check_team_member_model_access_denied_model(): proxy_logging_obj=MagicMock(), ) assert exc_info.value.type == ProxyErrorTypes.team_model_access_denied + assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index 2d1586f0b17..4ccde85dae2 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -140,6 +140,7 @@ async def test_handle_authentication_error_budget_exceeded(): ) assert exc_info.value.type == ProxyErrorTypes.budget_exceeded + assert int(exc_info.value.code) == status.HTTP_429_TOO_MANY_REQUESTS @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 95b3d746c66..50c5f43b218 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -1,6 +1,7 @@ import json import os import sys +from datetime import datetime, timedelta from types import SimpleNamespace from unittest.mock import ANY, AsyncMock, MagicMock, patch @@ -9,6 +10,7 @@ sys.path.insert( ) # Adds the parent directory to the system path import pytest +from fastapi import status import litellm import litellm.proxy.proxy_server @@ -178,6 +180,26 @@ async def test_custom_auth_does_not_enforce_key_model_access_by_default(): mock_can_key.assert_not_awaited() +@pytest.mark.asyncio +async def test_post_custom_auth_expired_key_returns_unauthorized(): + expired_token = UserAPIKeyAuth( + token="test_token", + expires=datetime.now() - timedelta(minutes=1), + ) + + with pytest.raises(ProxyException) as exc_info: + await _run_post_custom_auth_checks( + valid_token=expired_token, + request=MagicMock(), + request_data={}, + route="/v1/chat/completions", + parent_otel_span=None, + ) + + assert exc_info.value.type == ProxyErrorTypes.expired_key + assert int(exc_info.value.code) == status.HTTP_401_UNAUTHORIZED + + @pytest.mark.asyncio async def test_custom_auth_honors_key_level_model_access_restriction_allowed_with_opt_in(): valid_token = UserAPIKeyAuth(token="test_token", models=["gpt-4o-mini"]) @@ -934,6 +956,7 @@ async def test_proxy_admin_expired_key_from_cache(): assert ( exc_info.value.type == ProxyErrorTypes.expired_key ), f"Expected expired_key error type, got {exc_info.value.type}" + assert int(exc_info.value.code) == status.HTTP_401_UNAUTHORIZED assert "Expired Key" in str( exc_info.value.message ), f"Exception message should mention 'Expired Key', got: {exc_info.value.message}" diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py index 024d05e1037..8a3f9361ba1 100644 --- a/tests/test_openai_endpoints.py +++ b/tests/test_openai_endpoints.py @@ -303,7 +303,7 @@ async def test_chat_completion(): api_key=key_gen["key"], api_version="2024-02-15-preview", ) - with pytest.raises(openai.AuthenticationError) as e: + with pytest.raises(openai.PermissionDeniedError) as e: response = await azure_client.chat.completions.create( model="gpt-4", messages=[{"role": "user", "content": "Hello!"}], diff --git a/tests/test_users.py b/tests/test_users.py index 05253a19aa5..57fbb0483e4 100644 --- a/tests/test_users.py +++ b/tests/test_users.py @@ -302,14 +302,14 @@ async def test_user_model_access(): model="good-model", ) - with pytest.raises(openai.AuthenticationError): + with pytest.raises(openai.PermissionDeniedError): await chat_completion( session=session, key=key, model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", ) - with pytest.raises(openai.AuthenticationError): + with pytest.raises(openai.PermissionDeniedError): await chat_completion( session=session, key=key, From b888177ea67a83511cdd67fbd55b820059ac7e81 Mon Sep 17 00:00:00 2001 From: Michael-RZ-Berri Date: Sat, 9 May 2026 15:33:36 -0700 Subject: [PATCH 23/23] fix: reset proxy budget when initial reset duration is null then updated (#27488) Co-authored-by: Michael Riad Zaky --- litellm/proxy/proxy_server.py | 74 ++++++++++++++----- tests/test_litellm/proxy/test_proxy_server.py | 61 +++++++++++++++ 2 files changed, 117 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 02f9c9bef2f..493519f2328 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -211,6 +211,7 @@ from litellm import Router from litellm._logging import verbose_proxy_logger, verbose_router_logger from litellm.caching.caching import DualCache, RedisCache from litellm.caching.redis_cluster_cache import RedisClusterCache +from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.constants import ( _REALTIME_BODY_CACHE_SIZE, @@ -6750,27 +6751,64 @@ class ProxyStartupEvent: "budget_duration not set on Proxy. budget_duration is required to use max_budget." ) - # add proxy budget to db in the user table asyncio.create_task( - generate_key_helper_fn( # type: ignore - request_type="user", - table_name="user", - user_id=litellm_proxy_budget_name, - duration=None, - models=[], - aliases={}, - config={}, - spend=0, - max_budget=litellm.max_budget, - budget_duration=litellm.budget_duration, - query_type="update_data", - update_key_values={ - "max_budget": litellm.max_budget, - "budget_duration": litellm.budget_duration, - }, - ) + cls._upsert_proxy_budget_with_reset_at_backfill(litellm_proxy_budget_name) ) + @classmethod + async def _upsert_proxy_budget_with_reset_at_backfill( + cls, litellm_proxy_budget_name: str + ) -> None: + """ + Upsert the proxy admin user row with the configured max_budget / + budget_duration, then backfill budget_reset_at if currently NULL. + + The backfill uses `WHERE budget_reset_at IS NULL` so it only fires + when the row pre-existed without a reset schedule (e.g. row created + via a different path before the proxy budget was configured). On + subsequent restarts it no-ops, so an active reset window is never + slid forward. + """ + await generate_key_helper_fn( # type: ignore + request_type="user", + table_name="user", + user_id=litellm_proxy_budget_name, + duration=None, + models=[], + aliases={}, + config={}, + spend=0, + max_budget=litellm.max_budget, + budget_duration=litellm.budget_duration, + query_type="update_data", + update_key_values={ + "max_budget": litellm.max_budget, + "budget_duration": litellm.budget_duration, + }, + ) + + # Without this, the upsert leaves budget_reset_at=NULL on rows that + # took the UPDATE path, and reset_budget_for_litellm_users never + # matches them (NULL < now() is unknown in SQL) — so the proxy-wide + # spend cap blocks forever once it's hit. + if prisma_client is not None and litellm.budget_duration is not None: + try: + await prisma_client.db.litellm_usertable.update_many( + where={ + "user_id": litellm_proxy_budget_name, + "budget_reset_at": None, + }, + data={ + "budget_reset_at": get_budget_reset_time( + budget_duration=litellm.budget_duration + ) + }, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Failed to backfill budget_reset_at on proxy admin row: %s", e + ) + @classmethod async def _warm_global_spend_cache( cls, diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 6718f52cbf1..859594f7a0b 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -1728,6 +1728,67 @@ async def test_add_proxy_budget_to_db_only_creates_user_no_keys(): assert call_args.kwargs["query_type"] == "update_data" +@pytest.mark.asyncio +async def test_add_proxy_budget_to_db_backfills_budget_reset_at(): + """ + Test that _upsert_proxy_budget_with_reset_at_backfill issues a conditional + update_many with `WHERE budget_reset_at IS NULL` to backfill the column on + rows that pre-existed without a reset schedule. Without this, the proxy + admin row stays at NULL and reset_budget_for_litellm_users never matches + it (NULL < now() is unknown in SQL), so the global proxy budget never + resets. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + import litellm + from litellm.proxy.proxy_server import ProxyStartupEvent + + litellm.budget_duration = "30d" + litellm.max_budget = 100.0 + litellm_proxy_budget_name = "litellm-proxy-budget" + + mock_prisma = MagicMock() + mock_prisma.db.litellm_usertable.update_many = AsyncMock(return_value={"count": 1}) + + mock_generate_key_helper = AsyncMock( + return_value={ + "user_id": litellm_proxy_budget_name, + "max_budget": 100.0, + "budget_duration": "30d", + "spend": 0, + "models": [], + } + ) + + with ( + patch( + "litellm.proxy.proxy_server.generate_key_helper_fn", + mock_generate_key_helper, + ), + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + ): + await ProxyStartupEvent._upsert_proxy_budget_with_reset_at_backfill( + litellm_proxy_budget_name + ) + + # Upsert ran with the configured budget + mock_generate_key_helper.assert_called_once() + + # Backfill update_many ran with the conditional WHERE + mock_prisma.db.litellm_usertable.update_many.assert_called_once() + backfill_call = mock_prisma.db.litellm_usertable.update_many.call_args + assert backfill_call.kwargs["where"]["user_id"] == litellm_proxy_budget_name + assert backfill_call.kwargs["where"]["budget_reset_at"] is None + + # The backfilled value must be a real future datetime — anything else and + # reset_budget_for_litellm_users would still skip the row. + from datetime import datetime, timezone + + backfilled_reset_at = backfill_call.kwargs["data"]["budget_reset_at"] + assert isinstance(backfilled_reset_at, datetime) + assert backfilled_reset_at > datetime.now(timezone.utc) + + @pytest.mark.asyncio async def test_custom_ui_sso_sign_in_handler_config_loading(): """