From c119adb6dccedc1da78b30d91fa552e174e88191 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 23 Feb 2026 23:06:27 -0800 Subject: [PATCH 01/12] [Fix] UI - Virtual Keys: restrict Edit Settings button to key owners Non-owner Internal Users could see and interact with the "Edit Settings" button in the key Settings tab for keys they don't own. The button was gated by `rolesWithWriteAccess.includes(userRole)` (role-only check) instead of `canModifyKey` (ownership-aware), unlike the Regenerate and Delete buttons which already used the correct check. Replace the condition with `canModifyKey` so the Edit Settings button follows the same proxy-admin / team-admin / key-owner logic as the other action buttons. Add tests covering all permission paths. --- .../templates/key_info_view.test.tsx | 108 +++++++++++++++--- .../components/templates/key_info_view.tsx | 2 +- 2 files changed, 90 insertions(+), 20 deletions(-) diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx index db43d732e0a..bbb212e49b7 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx @@ -363,31 +363,101 @@ describe("KeyInfoView", () => { }); - it("should show edit button in settings tab when user has write access", async () => { - vi.mocked(useAuthorized).mockReturnValue({ - ...baseUseAuthorizedMock, - userRole: "Admin", + describe("'Edit Settings' button visibility in the Settings tab", () => { + const renderAndOpenSettingsTab = async (keyData = MOCK_KEY_DATA) => { + render( + {}} + keyId="test-key-id" + onKeyDataUpdate={() => {}} + teams={[]} + />, + ); + await waitFor(() => { + expect(screen.getByRole("tab", { name: /settings/i })).toBeInTheDocument(); + }); + await userEvent.click(screen.getByRole("tab", { name: /settings/i })); + }; + + it("should show the Edit Settings button when the user is a proxy admin for a key they do not own", async () => { + vi.mocked(useAuthorized).mockReturnValue({ + ...baseUseAuthorizedMock, + userId: "proxy-admin-user-id", + userRole: "proxy_admin", + }); + + await renderAndOpenSettingsTab({ ...MOCK_KEY_DATA, user_id: "someone-else-id" }); + + expect(screen.getByRole("button", { name: /edit settings/i })).toBeInTheDocument(); }); - render( - { }} - keyId={"test-key-id"} - onKeyDataUpdate={() => { }} - teams={[]} - />, - ); + it("should show the Edit Settings button when the user is the key owner", async () => { + vi.mocked(useAuthorized).mockReturnValue({ + ...baseUseAuthorizedMock, + userId: "owner-user-id", + userRole: "Internal User", + }); - await waitFor(() => { - const settingsTab = screen.getByRole("tab", { name: /settings/i }); - expect(settingsTab).toBeInTheDocument(); + await renderAndOpenSettingsTab({ ...MOCK_KEY_DATA, user_id: "owner-user-id" }); + + expect(screen.getByRole("button", { name: /edit settings/i })).toBeInTheDocument(); }); - const settingsTab = screen.getByRole("tab", { name: /settings/i }); - await userEvent.click(settingsTab); + it("should not show the Edit Settings button when an Internal User does not own the key", async () => { + vi.mocked(useAuthorized).mockReturnValue({ + ...baseUseAuthorizedMock, + userId: "non-owner-user-id", + userRole: "Internal User", + }); + + await renderAndOpenSettingsTab({ ...MOCK_KEY_DATA, user_id: "owner-user-id" }); + + expect(screen.queryByRole("button", { name: /edit settings/i })).not.toBeInTheDocument(); + }); + + it("should not show the Edit Settings button when the user is an Internal Viewer even if they own the key", async () => { + vi.mocked(useAuthorized).mockReturnValue({ + ...baseUseAuthorizedMock, + userId: "owner-user-id", + userRole: "Internal Viewer", + }); + + await renderAndOpenSettingsTab({ ...MOCK_KEY_DATA, user_id: "owner-user-id" }); + + expect(screen.queryByRole("button", { name: /edit settings/i })).not.toBeInTheDocument(); + }); + + it("should show the Edit Settings button when the user is a team admin for the key's team", async () => { + const teamId = "test-team-id"; + const teamAdminUserId = "team-admin-user"; + vi.mocked(useTeams).mockReturnValue({ + teams: [ + { + team_id: teamId, + team_alias: "Test Team", + models: [], + max_budget: null, + budget_duration: null, + tpm_limit: null, + rpm_limit: null, + organization_id: "org-1", + created_at: "2025-01-01T00:00:00Z", + keys: [], + members_with_roles: [{ user_id: teamAdminUserId, role: "admin" }], + spend: 0, + }, + ], + setTeams: vi.fn(), + }); + vi.mocked(useAuthorized).mockReturnValue({ + ...baseUseAuthorizedMock, + userId: teamAdminUserId, + userRole: "user", + }); + + await renderAndOpenSettingsTab({ ...MOCK_KEY_DATA, team_id: teamId, user_id: "other-user-id" }); - await waitFor(() => { expect(screen.getByRole("button", { name: /edit settings/i })).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index f2a12c90875..94ca90b9630 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -595,7 +595,7 @@ export default function KeyInfoView({
Key Settings - {!isEditing && userRole && rolesWithWriteAccess.includes(userRole) && ( + {!isEditing && canModifyKey && ( )}
From ef67b6b53363ea064deba1914568044b54020efa Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 24 Feb 2026 17:48:55 +0530 Subject: [PATCH 02/12] Add support for phase param --- litellm/types/responses/main.py | 3 + .../test_openai_responses_transformation.py | 316 +++++++++++++++++- 2 files changed, 318 insertions(+), 1 deletion(-) diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 8f6333ff900..bda53bae082 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -6,6 +6,7 @@ from typing_extensions import Any, List, Optional, TypedDict from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject +Phase = Optional[Literal["commentary", "final_answer"]] # TODO: Once openai sdk has updated, we can remove this and use the openai sdk type class GenericResponseOutputItemContentAnnotation(BaseLiteLLMOpenAIResponseObject): """Annotation for content in a message""" @@ -35,6 +36,7 @@ class OutputFunctionToolCall(BaseLiteLLMOpenAIResponseObject): type: Optional[str] # "function_call" id: Optional[str] status: Literal["in_progress", "completed", "incomplete"] + phase: Phase = None class OutputImageGenerationCall(BaseLiteLLMOpenAIResponseObject): @@ -57,6 +59,7 @@ class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject): status: str # "completed", "in_progress", etc. role: str # "assistant", "user", etc. content: List[OutputText] + phase: Phase = None class DeleteResponseResult(BaseLiteLLMOpenAIResponseObject): diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index 7c08716c04c..1a5ab808f7b 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -925,4 +925,318 @@ def test_get_supported_openai_params(): assert "temperature" in params assert "stream" in params assert "background" in params - assert "stream" in params \ No newline at end of file + assert "stream" in params + + +class TestPhaseParameter: + """Tests for the `phase` parameter on assistant output items (gpt-5.3-codex).""" + + def setup_method(self): + self.config = OpenAIResponsesAPIConfig() + self.model = "gpt-5.3-codex" + self.logging_obj = MagicMock() + + @staticmethod + def _make_output_text(text: str): + from litellm.types.responses.main import OutputText + + return OutputText(type="output_text", text=text, annotations=[]) + + def test_generic_response_output_item_accepts_phase_commentary(self): + from litellm.types.responses.main import GenericResponseOutputItem + + item = GenericResponseOutputItem( + type="message", + id="msg_001", + status="completed", + role="assistant", + content=[self._make_output_text("Thinking...")], + phase="commentary", + ) + assert item.phase == "commentary" + + def test_generic_response_output_item_accepts_phase_final_answer(self): + from litellm.types.responses.main import GenericResponseOutputItem + + item = GenericResponseOutputItem( + type="message", + id="msg_002", + status="completed", + role="assistant", + content=[self._make_output_text("The answer is 42.")], + phase="final_answer", + ) + assert item.phase == "final_answer" + + def test_generic_response_output_item_phase_defaults_to_none(self): + from litellm.types.responses.main import GenericResponseOutputItem + + item = GenericResponseOutputItem( + type="message", + id="msg_003", + status="completed", + role="assistant", + content=[self._make_output_text("Hello")], + ) + assert item.phase is None + + def test_output_function_tool_call_accepts_phase(self): + from litellm.types.responses.main import OutputFunctionToolCall + + item = OutputFunctionToolCall( + type="function_call", + id="fc_001", + arguments='{"query": "test"}', + call_id="call_001", + name="search", + status="completed", + phase="commentary", + ) + assert item.phase == "commentary" + + def test_input_passthrough_dict_preserves_phase(self): + """Dict input items (the normal HTTP flow) must preserve phase verbatim.""" + input_items = [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Hi"}], + }, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Preamble..."}], + "phase": "commentary", + }, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Done."}], + "phase": "final_answer", + }, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Neutral."}], + "phase": None, + }, + ] + + result = self.config._validate_input_param(input_items) + assert isinstance(result, list) + + assert "phase" not in result[0] + assert result[1]["phase"] == "commentary" + assert result[2]["phase"] == "final_answer" + assert result[3]["phase"] is None + + def test_input_passthrough_pydantic_preserves_non_null_phase(self): + """Pydantic input items must preserve non-null phase values.""" + from litellm.types.responses.main import GenericResponseOutputItem + + item = GenericResponseOutputItem( + type="message", + id="msg_010", + status="completed", + role="assistant", + content=[self._make_output_text("commentary")], + phase="commentary", + ) + + result = self.config._validate_input_param([item]) + assert isinstance(result, list) + assert result[0]["phase"] == "commentary" + + def test_response_parsing_preserves_phase_on_output(self): + """Non-streaming response must preserve phase on output items.""" + raw_json = { + "id": "resp_001", + "created_at": 1700000000, + "model": "gpt-5.3-codex", + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_001", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "preamble"}], + "phase": "commentary", + }, + { + "type": "message", + "id": "msg_002", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "answer"}], + "phase": "final_answer", + }, + ], + "usage": {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, + } + + response = ResponsesAPIResponse(**raw_json) + assert len(response.output) == 2 + + for idx, output_item in enumerate(response.output): + if isinstance(output_item, dict): + phase = output_item.get("phase") + else: + phase = getattr(output_item, "phase", None) + + expected = "commentary" if idx == 0 else "final_answer" + assert phase == expected, ( + f"output[{idx}] phase={phase!r}, expected {expected!r}" + ) + + def test_streaming_output_item_done_preserves_phase(self): + """OutputItemDoneEvent must preserve phase on its item.""" + from litellm.types.llms.openai import ( + OutputItemDoneEvent, + ResponsesAPIStreamEvents, + ) + + chunk = { + "type": "response.output_item.done", + "output_index": 0, + "sequence_number": 3, + "item": { + "type": "message", + "id": "msg_100", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "done"}], + "phase": "final_answer", + }, + } + + result = self.config.transform_streaming_response( + model=self.model, parsed_chunk=chunk, logging_obj=self.logging_obj + ) + + assert isinstance(result, OutputItemDoneEvent) + assert result.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE + assert getattr(result.item, "phase", None) == "final_answer" + + def test_streaming_output_item_added_preserves_phase(self): + """OutputItemAddedEvent must preserve phase on its item.""" + from litellm.types.llms.openai import ( + OutputItemAddedEvent, + ResponsesAPIStreamEvents, + ) + + chunk = { + "type": "response.output_item.added", + "output_index": 0, + "item": { + "type": "message", + "id": "msg_200", + "role": "assistant", + "phase": "commentary", + }, + } + + result = self.config.transform_streaming_response( + model=self.model, parsed_chunk=chunk, logging_obj=self.logging_obj + ) + + assert isinstance(result, OutputItemAddedEvent) + assert result.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + assert getattr(result.item, "phase", None) == "commentary" + + def test_streaming_response_completed_preserves_phase(self): + """ResponseCompletedEvent must preserve phase on output items inside the response.""" + completed_chunk = { + "type": "response.completed", + "response": { + "id": "resp_300", + "created_at": 1700000000, + "model": "gpt-5.3-codex", + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_300", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "final"}], + "phase": "final_answer", + } + ], + "usage": { + "input_tokens": 5, + "output_tokens": 10, + "total_tokens": 15, + }, + }, + } + + result = self.config.transform_streaming_response( + model=self.model, + parsed_chunk=completed_chunk, + logging_obj=self.logging_obj, + ) + + assert result.type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + output_item = result.response.output[0] + if isinstance(output_item, dict): + assert output_item["phase"] == "final_answer" + else: + assert getattr(output_item, "phase", None) == "final_answer" + + def test_phase_roundtrip_output_to_input(self): + """Simulate full round-trip: parse response output, then send items back as input.""" + raw_json = { + "id": "resp_rt", + "created_at": 1700000000, + "model": "gpt-5.3-codex", + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_rt1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "preamble"}], + "phase": "commentary", + }, + { + "type": "message", + "id": "msg_rt2", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "answer"}], + "phase": "final_answer", + }, + ], + "usage": {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, + } + + response = ResponsesAPIResponse(**raw_json) + + input_items = [] + for item in response.output: + if isinstance(item, dict): + input_items.append(item) + else: + input_items.append( + item.model_dump() if hasattr(item, "model_dump") else dict(item) + ) + + input_items.append( + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "next question"}], + } + ) + + validated = self.config._validate_input_param(input_items) + assert isinstance(validated, list) + + assert validated[0]["phase"] == "commentary" + assert validated[1]["phase"] == "final_answer" + assert "phase" not in validated[2] \ No newline at end of file From ac720defc3d7ae338299833055474dedfe667387 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 24 Feb 2026 17:50:38 +0530 Subject: [PATCH 03/12] Add documentation related to phase --- docs/my-website/blog/gpt_5_3_codex/index.md | 145 ++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 docs/my-website/blog/gpt_5_3_codex/index.md diff --git a/docs/my-website/blog/gpt_5_3_codex/index.md b/docs/my-website/blog/gpt_5_3_codex/index.md new file mode 100644 index 00000000000..321e573cccf --- /dev/null +++ b/docs/my-website/blog/gpt_5_3_codex/index.md @@ -0,0 +1,145 @@ +--- +slug: gpt_5_3_codex +title: "Day 0 Support: GPT-5.3-Codex" +date: 2026-02-24T10:00:00 +authors: + - name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +description: "Day 0 support for GPT-5.3-Codex on LiteLLM, including phase parameter handling for Responses API." +tags: [openai, gpt-5.3-codex, codex, day 0 support] +hide_table_of_contents: false +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +LiteLLM now supports GPT-5.3-Codex on Day 0, including support for the new assistant `phase` metadata on Responses API output items. + +## Why `phase` matters for GPT-5.3-Codex + +`phase` appears on assistant output items and helps distinguish preamble/commentary turns from final closeout responses. + +Reference: [Phase parameter docs](https://developers-site-git-alphas-venusaur-api-openai.vercel.app/alphas/venusaur-api/phase-parameter) + +Supported values: +- `null` +- `"commentary"` +- `"final_answer"` + +Important: +- Persist assistant output items with `phase` exactly as returned. +- Send those assistant items back on the next turn. +- Do **not** add `phase` to user messages. + +## Docker Image + +```bash +docker pull ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 +``` + +## Usage + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: gpt-5.3-codex + litellm_params: + model: openai/gpt-5.3-codex +``` + +**2. Start the proxy** + +```bash +docker run -d \ + -p 4000:4000 \ + -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -v $(pwd)/config.yaml:/app/config.yaml \ + ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \ + --config /app/config.yaml +``` + + +**3. Test it** + +```bash +curl -X POST "http://0.0.0.0:4000/v1/responses" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{ + "model": "gpt-5.3-codex", + "input": "Write a Python script that checks if a number is prime." + }' +``` + + + + +## Python Example: Persist `phase` with OpenAI Client + LiteLLM Base URL + +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://0.0.0.0:4000/v1", # LiteLLM Proxy + api_key="your-litellm-api-key", +) + +items = [] # Persist this per conversation/thread + + +def _item_get(item, key, default=None): + if isinstance(item, dict): + return item.get(key, default) + return getattr(item, key, default) + + +def run_turn(user_text: str): + global items + + # User message: no phase field + items.append( + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": user_text}], + } + ) + + resp = client.responses.create( + model="gpt-5.3-codex", + input=items, + ) + + # Persist assistant output items verbatim, including phase + for out_item in (resp.output or []): + items.append(out_item) + + # Optional: inspect latest phase for UI/telemetry routing + latest_phase = None + for out_item in reversed(resp.output or []): + if _item_get(out_item, "type") == "output_item.done" and _item_get(out_item, "phase") is not None: + latest_phase = _item_get(out_item, "phase") + break + + return resp, latest_phase +``` + +## Notes + +- Use `/v1/responses` for GPT Codex models. +- Preserve full assistant output history for best multi-turn behavior. +- If `phase` metadata is dropped during history reconstruction, output quality can degrade on long-running tasks. From 3e6c10a0710e7c64f27f5875bbf1a30702b4f143 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Tue, 24 Feb 2026 19:40:09 +0530 Subject: [PATCH 04/12] security: fix critical/high CVEs in OS-level libs and NPM transitive --- Dockerfile | 20 ++++++++-- ci_cd/security_scans.sh | 1 + docker/Dockerfile.custom_ui | 23 +++++++++++- docker/Dockerfile.database | 20 ++++++++-- docker/Dockerfile.dev | 37 ++++++++++++++++--- docker/Dockerfile.non_root | 23 +++++++++--- docs/my-website/package.json | 13 ++++++- litellm-js/spend-logs/package.json | 18 +++++++-- package.json | 18 +++++++-- requirements.txt | 4 +- tests/proxy_admin_ui_tests/package.json | 18 +++++++-- .../ui_unit_tests/package.json | 16 +++++++- ui/litellm-dashboard/package.json | 17 +++++++-- 13 files changed, 191 insertions(+), 37 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5e93a0c627e..83d3640e763 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,7 @@ USER root # Install runtime dependencies (libsndfile needed for audio processing on ARM64) RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \ - npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \ + npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \ # SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested # levels inside its dependency tree. `npm install -g ` only creates a # SEPARATE global package, it does NOT replace npm's internal copies. @@ -64,6 +64,12 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ done && \ + find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ + done && \ + find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ + done && \ npm cache clean --force WORKDIR /app @@ -90,14 +96,20 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \ # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/. # Patch every copy of tar, glob, and brace-expansion inside that tree. RUN GLOBAL="$(npm root -g)" && \ - find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \ + find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ done && \ - find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \ + find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ done && \ - find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \ + find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ + done && \ + find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ + done && \ + find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ done # Install semantic_router and aurelio-sdk using script diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index b60e7ec5235..0e50f15d043 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -160,6 +160,7 @@ run_grype_scans() { "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time "GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code "GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code + "CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up ) # Build JSON array of allowlisted CVE IDs for jq diff --git a/docker/Dockerfile.custom_ui b/docker/Dockerfile.custom_ui index 177d7b7b12a..fb98846a6cc 100644 --- a/docker/Dockerfile.custom_ui +++ b/docker/Dockerfile.custom_ui @@ -5,8 +5,21 @@ FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev WORKDIR /app # Install Node.js and npm (adjust version as needed) -RUN apt-get update && apt-get install -y nodejs npm && \ - npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \ +RUN apt-get update && apt-get upgrade -y \ + libxml2 \ + libexpat1 \ + openssl \ + libssl3 \ + git \ + libkrb5-3 \ + libglib2.0-0 \ + wget \ + libaom3 \ + libxslt1.1 \ + libgnutls30 \ + libc6 && \ + apt-get install -y nodejs npm && \ + npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \ GLOBAL="$(npm root -g)" && \ find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ @@ -17,6 +30,12 @@ RUN apt-get update && apt-get install -y nodejs npm && \ find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ done && \ + find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ + done && \ + find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ + done && \ npm cache clean --force # Copy the UI source into the container diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index a6fcd98ab6d..371766bd9db 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -50,7 +50,7 @@ USER root # Install runtime dependencies RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \ - npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \ + npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \ GLOBAL="$(npm root -g)" && \ find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ @@ -61,6 +61,12 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ done && \ + find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ + done && \ + find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ + done && \ npm cache clean --force WORKDIR /app @@ -79,14 +85,20 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/. # Patch every copy of tar, glob, and brace-expansion inside that tree. RUN GLOBAL="$(npm root -g)" && \ - find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \ + find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ done && \ - find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \ + find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ done && \ - find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \ + find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ + done && \ + find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ + done && \ + find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ done # Install semantic_router and aurelio-sdk using script diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index bc1d22d5e05..a5312dec9e3 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -56,13 +56,26 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root # Install only runtime dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - libssl3 \ +RUN apt-get update && apt-get upgrade -y \ + libxml2 \ + libexpat1 \ + openssl \ + libssl3 \ + git \ + libkrb5-3 \ + libglib2.0-0 \ + wget \ + libaom3 \ + libxslt1.1 \ + libgnutls30 \ + libc6 \ + && apt-get install -y --no-install-recommends \ + libssl3 \ libatomic1 \ nodejs \ npm \ && rm -rf /var/lib/apt/lists/* \ - && npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \ + && npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 \ && GLOBAL="$(npm root -g)" \ && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ @@ -73,6 +86,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ done \ + && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ + done \ + && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ + done \ && npm cache clean --force WORKDIR /app @@ -95,14 +114,20 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/ # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/. # Patch every copy of tar, glob, and brace-expansion inside that tree. RUN GLOBAL="$(npm root -g)" && \ - find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \ + find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ done && \ - find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \ + find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ done && \ - find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \ + find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ + done && \ + find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ + done && \ + find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ done # Generate prisma client and set permissions diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 004377e19b3..fda591df083 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -80,7 +80,7 @@ ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ XDG_CACHE_HOME=/app/.cache \ PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}" -RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.12.0 \ +RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.13.1 \ && mkdir -p /app/.cache/npm RUN NPM_CONFIG_CACHE=/app/.cache/npm \ @@ -105,7 +105,8 @@ RUN for i in 1 2 3; do \ && for i in 1 2 3; do \ apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \ done \ - && npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \ + && apk upgrade --no-cache nodejs \ + && npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 \ && GLOBAL="$(npm root -g)" \ && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ @@ -116,6 +117,12 @@ RUN for i in 1 2 3; do \ && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ done \ + && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ + done \ + && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ + done \ && npm cache clean --force # Copy artifacts from builder @@ -162,14 +169,20 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \ # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/. # Patch every copy of tar, glob, and brace-expansion inside that tree. RUN GLOBAL="$(npm root -g)" && \ - find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \ + find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ done && \ - find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \ + find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ done && \ - find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \ + find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ + done && \ + find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ + done && \ + find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ done # Permissions, cleanup, and Prisma prep diff --git a/docs/my-website/package.json b/docs/my-website/package.json index c4fa04c96a7..2dad6d0f16b 100644 --- a/docs/my-website/package.json +++ b/docs/my-website/package.json @@ -62,6 +62,8 @@ "gray-matter": "4.0.3", "glob": ">=11.1.0", "tar": ">=7.5.8", + "minimatch": ">=10.2.1", + "diff": ">=8.0.3", "@isaacs/brace-expansion": ">=5.0.1", "node-forge": ">=1.3.2", "mdast-util-to-hast": ">=13.2.1", @@ -81,6 +83,15 @@ "url-loader": { "ajv": "6.14.0" }, - "minimatch": "10.2.1" + "@babel/traverse": ">=7.23.2", + "ws": ">=7.5.10", + "http-proxy-middleware": ">=2.0.9", + "tar-fs": ">=2.1.4", + "webpack-dev-middleware": ">=5.3.4", + "braces": ">=3.0.3", + "axios": ">=0.30.2", + "webpack": ">=5.94.0", + "serve-static": ">=1.16.0", + "path-to-regexp": ">=0.1.12" } } \ No newline at end of file diff --git a/litellm-js/spend-logs/package.json b/litellm-js/spend-logs/package.json index 67292567145..5a7a08cb9ef 100644 --- a/litellm-js/spend-logs/package.json +++ b/litellm-js/spend-logs/package.json @@ -12,7 +12,19 @@ }, "overrides": { "glob": ">=11.1.0", - "tar": ">=7.5.7", - "@isaacs/brace-expansion": ">=5.0.1" + "tar": ">=7.5.8", + "minimatch": ">=10.2.1", + "diff": ">=8.0.3", + "@isaacs/brace-expansion": ">=5.0.1", + "@babel/traverse": ">=7.23.2", + "ws": ">=7.5.10", + "http-proxy-middleware": ">=2.0.9", + "tar-fs": ">=2.1.4", + "webpack-dev-middleware": ">=5.3.4", + "braces": ">=3.0.3", + "axios": ">=0.30.2", + "webpack": ">=5.94.0", + "serve-static": ">=1.16.0", + "path-to-regexp": ">=0.1.12" } -} +} \ No newline at end of file diff --git a/package.json b/package.json index ab9e15f46a7..a45e116b277 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,19 @@ }, "overrides": { "glob": ">=11.1.0", - "tar": ">=7.5.7", - "@isaacs/brace-expansion": ">=5.0.1" + "tar": ">=7.5.8", + "minimatch": ">=10.2.1", + "diff": ">=8.0.3", + "@isaacs/brace-expansion": ">=5.0.1", + "@babel/traverse": ">=7.23.2", + "ws": ">=7.5.10", + "http-proxy-middleware": ">=2.0.9", + "tar-fs": ">=2.1.4", + "webpack-dev-middleware": ">=5.3.4", + "braces": ">=3.0.3", + "axios": ">=0.30.2", + "webpack": ">=5.94.0", + "serve-static": ">=1.16.0", + "path-to-regexp": ">=0.1.12" } -} +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 832c3f75b72..dbde6ababc9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,8 @@ urllib3>=2.6.0 # CVE-2025-66471, CVE-2025-66418, CVE-2026-21441 tornado>=6.5.3 # CVE-2025-67725, CVE-2025-67726, CVE-2025-67724 filelock>=3.20.1 # CVE-2025-68146 +h11>=0.16.0 # CVE-2025-43859, GHSA-vqfr-h8mv-ghfj — HTTP request smuggling +wheel>=0.46.2 # CVE-2026-24049 — path traversal Pillow==12.1.1 #GHSA-cfh3-3jmp-rvhc cryptography==46.0.5 #GHSA-r6ph-v2qm-q3c2 @@ -21,7 +23,7 @@ boto3==1.40.53 # aws bedrock/sagemaker calls (has bedrock-agentcore-control, com redis==5.2.1 # redis caching redisvl==0.4.1 ## redis semantic caching prisma==0.11.0 # for db -nodejs-wheel-binaries==24.12.0 ## required by prisma for migrations, prevents runtime download (updated from nodejs-bin for security fixes) +nodejs-wheel-binaries==24.13.1 ## required by prisma for migrations, prevents runtime download (updated from nodejs-bin for security fixes) mangum==0.17.0 # for aws lambda functions pynacl==1.6.2 # for encrypting keys google-cloud-aiplatform==1.133.0 # for vertex ai calls diff --git a/tests/proxy_admin_ui_tests/package.json b/tests/proxy_admin_ui_tests/package.json index 48de2c1dba2..037ec7082a7 100644 --- a/tests/proxy_admin_ui_tests/package.json +++ b/tests/proxy_admin_ui_tests/package.json @@ -13,7 +13,19 @@ }, "overrides": { "glob": ">=11.1.0", - "tar": ">=7.5.7", - "@isaacs/brace-expansion": ">=5.0.1" + "tar": ">=7.5.8", + "minimatch": ">=10.2.1", + "diff": ">=8.0.3", + "@isaacs/brace-expansion": ">=5.0.1", + "@babel/traverse": ">=7.23.2", + "ws": ">=7.5.10", + "http-proxy-middleware": ">=2.0.9", + "tar-fs": ">=2.1.4", + "webpack-dev-middleware": ">=5.3.4", + "braces": ">=3.0.3", + "axios": ">=0.30.2", + "webpack": ">=5.94.0", + "serve-static": ">=1.16.0", + "path-to-regexp": ">=0.1.12" } -} +} \ No newline at end of file diff --git a/tests/proxy_admin_ui_tests/ui_unit_tests/package.json b/tests/proxy_admin_ui_tests/ui_unit_tests/package.json index 4c7d7addf0e..eb9c7473a5b 100644 --- a/tests/proxy_admin_ui_tests/ui_unit_tests/package.json +++ b/tests/proxy_admin_ui_tests/ui_unit_tests/package.json @@ -25,7 +25,19 @@ }, "overrides": { "glob": ">=11.1.0", - "tar": ">=7.5.7", - "@isaacs/brace-expansion": ">=5.0.1" + "tar": ">=7.5.8", + "minimatch": ">=10.2.1", + "diff": ">=8.0.3", + "@isaacs/brace-expansion": ">=5.0.1", + "@babel/traverse": ">=7.23.2", + "ws": ">=7.5.10", + "http-proxy-middleware": ">=2.0.9", + "tar-fs": ">=2.1.4", + "webpack-dev-middleware": ">=5.3.4", + "braces": ">=3.0.3", + "axios": ">=0.30.2", + "webpack": ">=5.94.0", + "serve-static": ">=1.16.0", + "path-to-regexp": ">=0.1.12" } } \ No newline at end of file diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index b05d707d5ab..567673c0989 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -86,14 +86,25 @@ "mermaid": ">=11.10.0", "js-yaml": ">=4.1.1", "glob": ">=11.1.0", - "tar": ">=7.5.7", + "tar": ">=7.5.8", + "minimatch": ">=10.2.1", "@isaacs/brace-expansion": ">=5.0.1", "node-forge": ">=1.3.2", "lodash-es": ">=4.17.23", - "lodash": ">=4.17.23" + "lodash": ">=4.17.23", + "@babel/traverse": ">=7.23.2", + "ws": ">=7.5.10", + "http-proxy-middleware": ">=2.0.9", + "tar-fs": ">=2.1.4", + "webpack-dev-middleware": ">=5.3.4", + "braces": ">=3.0.3", + "axios": ">=0.30.2", + "webpack": ">=5.94.0", + "serve-static": ">=1.16.0", + "path-to-regexp": ">=0.1.12" }, "engines": { "node": ">=18.17.0", "npm": ">=8.3.0" } -} +} \ No newline at end of file From 4652c73259e2e3b1d3cadea360f2bc5f4b27d1eb Mon Sep 17 00:00:00 2001 From: Sean Marsh Glover Date: Tue, 24 Feb 2026 11:16:59 -0500 Subject: [PATCH 05/12] feat(proxy): limit concurrent health checks with health_check_concurrency (#20584) * staged first pass * black * Update litellm/proxy/health_check.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * simpler * restore cached logo * fix tests for perform_health_check max_concurrency arg * implement pr suggestion * and the helm chart * add configureable resources and probes to the deployment in the helm chart * more helm chart unittests * move some background healthcheck loggin to debug --------- Co-authored-by: Sean Glover Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- deploy/charts/litellm-helm/README.md | 4 + .../templates/configmap-litellm.yaml | 2 +- .../litellm-helm/templates/deployment.yaml | 25 +- .../litellm-helm/tests/deployment_tests.yaml | 148 ++++++++++- deploy/charts/litellm-helm/values.yaml | 25 ++ litellm/proxy/_types.py | 7 + litellm/proxy/health_check.py | 194 ++++++++++++-- .../shared_health_check_manager.py | 96 +++---- .../health_endpoints/_health_endpoints.py | 114 ++++---- litellm/proxy/proxy_server.py | 247 ++++++++++++++---- .../litellm_utils_tests/test_health_check.py | 119 ++++++++- tests/proxy_unit_tests/test_proxy_server.py | 8 +- 12 files changed, 809 insertions(+), 180 deletions(-) diff --git a/deploy/charts/litellm-helm/README.md b/deploy/charts/litellm-helm/README.md index 2fa856843f3..74e70f4aeb4 100644 --- a/deploy/charts/litellm-helm/README.md +++ b/deploy/charts/litellm-helm/README.md @@ -36,6 +36,10 @@ If `db.useStackgresOperator` is used (not yet implemented): | `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` | | `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` | | `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` | +| `livenessProbe.*` | Liveness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` | +| `readinessProbe.*` | Readiness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` | +| `startupProbe.*` | Startup probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` | +| `resources.*` | CPU/memory requests and limits for the LiteLLM container. | `{}` | | `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` | | `ingress.labels` | Additional labels for the Ingress resource | `{}` | | `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A | diff --git a/deploy/charts/litellm-helm/templates/configmap-litellm.yaml b/deploy/charts/litellm-helm/templates/configmap-litellm.yaml index cf35917da03..acbe4e3a4b5 100644 --- a/deploy/charts/litellm-helm/templates/configmap-litellm.yaml +++ b/deploy/charts/litellm-helm/templates/configmap-litellm.yaml @@ -6,4 +6,4 @@ metadata: data: config.yaml: | {{ .Values.proxy_config | toYaml | indent 6 }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 4ac5582d060..df483ab927d 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -158,18 +158,31 @@ spec: {{- end }} livenessProbe: httpGet: - path: /health/liveliness + path: {{ .Values.livenessProbe.path | quote }} port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} readinessProbe: httpGet: - path: /health/readiness + path: {{ .Values.readinessProbe.path | quote }} port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} startupProbe: httpGet: - path: /health/readiness + path: {{ .Values.startupProbe.path | quote }} port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }} - failureThreshold: 30 - periodSeconds: 10 + initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.startupProbe.periodSeconds }} + timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }} + successThreshold: {{ .Values.startupProbe.successThreshold }} + failureThreshold: {{ .Values.startupProbe.failureThreshold }} resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: @@ -235,4 +248,4 @@ spec: {{- if .Values.topologySpreadConstraints }} topologySpreadConstraints: {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/deploy/charts/litellm-helm/tests/deployment_tests.yaml b/deploy/charts/litellm-helm/tests/deployment_tests.yaml index f1229e10235..2e9c48043de 100644 --- a/deploy/charts/litellm-helm/tests/deployment_tests.yaml +++ b/deploy/charts/litellm-helm/tests/deployment_tests.yaml @@ -159,4 +159,150 @@ tests: value: -c - equal: path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2] - value: echo "Container stopping" \ No newline at end of file + value: echo "Container stopping" + - it: should render background health check settings from proxy_config.general_settings + template: configmap-litellm.yaml + set: + proxy_config.general_settings.background_health_checks: true + proxy_config.general_settings.health_check_interval: 240 + proxy_config.general_settings.health_check_concurrency: 16 + proxy_config.general_settings.health_check_details: false + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: '(?m)^\s*background_health_checks:\s*true$' + - matchRegex: + path: data["config.yaml"] + pattern: '(?m)^\s*health_check_interval:\s*240$' + - matchRegex: + path: data["config.yaml"] + pattern: '(?m)^\s*health_check_concurrency:\s*16$' + - matchRegex: + path: data["config.yaml"] + pattern: '(?m)^\s*health_check_details:\s*false$' + - it: should allow overriding liveness, readiness, and startup probes + template: deployment.yaml + set: + livenessProbe: + path: /custom/livez + initialDelaySeconds: 5 + periodSeconds: 15 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + path: /custom/readyz + initialDelaySeconds: 10 + periodSeconds: 20 + timeoutSeconds: 6 + successThreshold: 1 + failureThreshold: 6 + startupProbe: + path: /custom/startupz + initialDelaySeconds: 15 + periodSeconds: 25 + timeoutSeconds: 7 + successThreshold: 1 + failureThreshold: 40 + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /custom/livez + - equal: + path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds + value: 5 + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /custom/readyz + - equal: + path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds + value: 6 + - equal: + path: spec.template.spec.containers[0].startupProbe.httpGet.path + value: /custom/startupz + - equal: + path: spec.template.spec.containers[0].startupProbe.failureThreshold + value: 40 + - it: should render container resources from values + template: deployment.yaml + set: + resources: + limits: + cpu: 500m + memory: 2Gi + requests: + cpu: 250m + memory: 1Gi + asserts: + - equal: + path: spec.template.spec.containers[0].resources.limits.cpu + value: 500m + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 2Gi + - equal: + path: spec.template.spec.containers[0].resources.requests.cpu + value: 250m + - equal: + path: spec.template.spec.containers[0].resources.requests.memory + value: 1Gi + - it: should keep default probes and empty resources unchanged + template: deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /health/liveliness + - equal: + path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds + value: 0 + - equal: + path: spec.template.spec.containers[0].livenessProbe.periodSeconds + value: 10 + - equal: + path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds + value: 1 + - equal: + path: spec.template.spec.containers[0].livenessProbe.successThreshold + value: 1 + - equal: + path: spec.template.spec.containers[0].livenessProbe.failureThreshold + value: 3 + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /health/readiness + - equal: + path: spec.template.spec.containers[0].readinessProbe.initialDelaySeconds + value: 0 + - equal: + path: spec.template.spec.containers[0].readinessProbe.periodSeconds + value: 10 + - equal: + path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds + value: 1 + - equal: + path: spec.template.spec.containers[0].readinessProbe.successThreshold + value: 1 + - equal: + path: spec.template.spec.containers[0].readinessProbe.failureThreshold + value: 3 + - equal: + path: spec.template.spec.containers[0].startupProbe.httpGet.path + value: /health/readiness + - equal: + path: spec.template.spec.containers[0].startupProbe.initialDelaySeconds + value: 0 + - equal: + path: spec.template.spec.containers[0].startupProbe.periodSeconds + value: 10 + - equal: + path: spec.template.spec.containers[0].startupProbe.timeoutSeconds + value: 1 + - equal: + path: spec.template.spec.containers[0].startupProbe.successThreshold + value: 1 + - equal: + path: spec.template.spec.containers[0].startupProbe.failureThreshold + value: 30 + - equal: + path: spec.template.spec.containers[0].resources + value: {} diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index cea25974bb0..d62f5b29c2b 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -84,6 +84,31 @@ service: separateHealthApp: false separateHealthPort: 8081 +# Probe tuning for proxy container +livenessProbe: + path: /health/liveliness + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + +readinessProbe: + path: /health/readiness + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + +startupProbe: + path: /health/readiness + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 30 + ingress: enabled: false className: "nginx" diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 95739834a9a..f354e28acd7 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2079,6 +2079,13 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): health_check_interval: int = Field( 300, description="background health check interval in seconds" ) + health_check_concurrency: Optional[int] = Field( + None, + description=( + "limit concurrent health checks per cycle; when unset, " + "health checks run without a concurrency cap" + ), + ) alerting: Optional[List] = Field( None, description="List of alerting integrations. Today, just slack - `alerting: ['slack']`", diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 427a16a9801..4777f64405d 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -3,6 +3,9 @@ import asyncio import logging import random +import sys +import threading +import time from typing import List, Optional import litellm @@ -23,6 +26,29 @@ ILLEGAL_DISPLAY_PARAMS = [ MINIMAL_DISPLAY_PARAMS = ["model", "mode_error"] +def _get_process_rss_mb() -> Optional[float]: + """ + Get process RSS memory in MB. + On Linux, ru_maxrss is in KB. On macOS, ru_maxrss is in bytes. + """ + try: + import resource + + ru_maxrss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + if sys.platform == "darwin": + return float(ru_maxrss) / (1024 * 1024) + return float(ru_maxrss) / 1024 + except Exception: + return None + + +def _rss_mb_for_log() -> str: + rss_mb = _get_process_rss_mb() + if rss_mb is None: + return "unknown" + return f"{rss_mb:.2f}" + + def _get_random_llm_message(): """ Get a random message from the LLM. @@ -67,26 +93,29 @@ async def run_with_timeout(task, timeout): try: return await asyncio.wait_for(task, timeout) except asyncio.TimeoutError: - task.cancel() - # Only cancel child tasks of the current task - current_task = asyncio.current_task() - for t in asyncio.all_tasks(): - if t != current_task: - t.cancel() - try: - await asyncio.wait_for(task, 0.1) # Give 100ms for cleanup - except (asyncio.TimeoutError, asyncio.CancelledError, Exception): - pass + # `asyncio.wait_for()` already cancels only the awaited task on timeout. + # Do not cancel unrelated sibling health check tasks. return {"error": "Timeout exceeded"} -async def _perform_health_check(model_list: list, details: Optional[bool] = True): +async def _perform_health_check( + model_list: list, + details: Optional[bool] = True, + max_concurrency: Optional[int] = None, + instrumentation_context: Optional[dict] = None, +): """ Perform a health check for each model in the list. + + max_concurrency: Optional limit on concurrent health check requests. """ - tasks = [] - for model in model_list: + instrumentation_context = instrumentation_context or {} + instrumentation_enabled = bool(instrumentation_context.get("enabled", False)) + cycle_id = instrumentation_context.get("cycle_id", "unknown") + source = instrumentation_context.get("source", "unknown") + + async def _run_model_health_check(model: dict): litellm_params = model["litellm_params"] model_info = model.get("model_info", {}) mode = model_info.get("mode", None) @@ -95,9 +124,9 @@ async def _perform_health_check(model_list: list, details: Optional[bool] = True ) timeout = model_info.get("health_check_timeout") or HEALTH_CHECK_TIMEOUT_SECONDS - task = run_with_timeout( + return await run_with_timeout( litellm.ahealth_check( - model["litellm_params"], + litellm_params, mode=mode, prompt=DEFAULT_HEALTH_CHECK_PROMPT, input=["test from litellm"], @@ -105,9 +134,73 @@ async def _perform_health_check(model_list: list, details: Optional[bool] = True timeout, ) - tasks.append(task) + async def _run_health_checks_with_bounded_concurrency( + models: list, concurrency_limit: int + ) -> tuple[list, int]: + """ + Run health checks with at most `concurrency_limit` active tasks. + Preserves result ordering to match `models`. + """ + results: list = [None] * len(models) + tasks_to_index: dict[asyncio.Task, int] = {} + model_iter = iter(enumerate(models)) + peak_in_flight = 0 - results = await asyncio.gather(*tasks, return_exceptions=True) + def _schedule_next() -> bool: + nonlocal peak_in_flight + try: + idx, next_model = next(model_iter) + except StopIteration: + return False + task = asyncio.create_task(_run_model_health_check(next_model)) + tasks_to_index[task] = idx + peak_in_flight = max(peak_in_flight, len(tasks_to_index)) + return True + + for _ in range(min(concurrency_limit, len(models))): + _schedule_next() + + while tasks_to_index: + done, _ = await asyncio.wait( + set(tasks_to_index.keys()), + return_when=asyncio.FIRST_COMPLETED, + ) + for task in done: + idx = tasks_to_index.pop(task) + try: + results[idx] = task.result() + except Exception as e: + results[idx] = e + _schedule_next() + + return results, peak_in_flight + + dispatch_mode = "unbounded" + peak_in_flight = 0 + if isinstance(max_concurrency, int) and max_concurrency > 0: + dispatch_mode = "bounded" + results, peak_in_flight = await _run_health_checks_with_bounded_concurrency( + model_list, max_concurrency + ) + else: + tasks = [ + asyncio.create_task(_run_model_health_check(model)) for model in model_list + ] + peak_in_flight = len(tasks) + results = await asyncio.gather(*tasks, return_exceptions=True) + + if instrumentation_enabled: + logger.debug( + "health_check_dispatch_summary source=%s cycle_id=%s mode=%s model_count=%d max_concurrency=%s peak_in_flight=%d thread_count=%d rss_mb=%s", + source, + cycle_id, + dispatch_mode, + len(model_list), + max_concurrency, + peak_in_flight, + threading.active_count(), + _rss_mb_for_log(), + ) healthy_endpoints = [] unhealthy_endpoints = [] @@ -190,6 +283,8 @@ async def perform_health_check( model: Optional[str] = None, cli_model: Optional[str] = None, details: Optional[bool] = True, + max_concurrency: Optional[int] = None, + instrumentation_context: Optional[dict] = None, ): """ Perform a health check on the system. @@ -197,14 +292,28 @@ async def perform_health_check( Returns: (bool): True if the health check passes, False otherwise. """ + instrumentation_context = instrumentation_context or {} + instrumentation_enabled = bool(instrumentation_context.get("enabled", False)) + cycle_id = instrumentation_context.get("cycle_id", "unknown") + source = instrumentation_context.get("source", "unknown") + if not model_list: if cli_model: model_list = [ {"model_name": cli_model, "litellm_params": {"model": cli_model}} ] else: + if instrumentation_enabled: + logger.debug( + "health_check_cycle_skipped source=%s cycle_id=%s reason=no_models", + source, + cycle_id, + ) return [], [] + cycle_start_time = time.monotonic() + requested_model_count = len(model_list) + if model is not None: _new_model_list = [ x for x in model_list if x["litellm_params"]["model"] == model @@ -213,11 +322,56 @@ async def perform_health_check( _new_model_list = [x for x in model_list if x["model_name"] == model] model_list = _new_model_list + post_filter_model_count = len(model_list) model_list = filter_deployments_by_id( model_list=model_list ) # filter duplicate deployments (e.g. when model alias'es are used) - healthy_endpoints, unhealthy_endpoints = await _perform_health_check( - model_list, details - ) + deduped_model_count = len(model_list) + + if instrumentation_enabled: + logger.debug( + "health_check_cycle_start source=%s cycle_id=%s requested_model_count=%d post_model_filter_count=%d deduped_model_count=%d max_concurrency=%s thread_count=%d rss_mb=%s", + source, + cycle_id, + requested_model_count, + post_filter_model_count, + deduped_model_count, + max_concurrency, + threading.active_count(), + _rss_mb_for_log(), + ) + + try: + healthy_endpoints, unhealthy_endpoints = await _perform_health_check( + model_list, + details, + max_concurrency=max_concurrency, + instrumentation_context=instrumentation_context, + ) + except Exception: + if instrumentation_enabled: + logger.exception( + "health_check_cycle_failed source=%s cycle_id=%s model_count=%d duration_ms=%.2f thread_count=%d rss_mb=%s", + source, + cycle_id, + deduped_model_count, + (time.monotonic() - cycle_start_time) * 1000, + threading.active_count(), + _rss_mb_for_log(), + ) + raise + + if instrumentation_enabled: + logger.debug( + "health_check_cycle_complete source=%s cycle_id=%s model_count=%d healthy_count=%d unhealthy_count=%d duration_ms=%.2f thread_count=%d rss_mb=%s", + source, + cycle_id, + deduped_model_count, + len(healthy_endpoints), + len(unhealthy_endpoints), + (time.monotonic() - cycle_start_time) * 1000, + threading.active_count(), + _rss_mb_for_log(), + ) return healthy_endpoints, unhealthy_endpoints diff --git a/litellm/proxy/health_check_utils/shared_health_check_manager.py b/litellm/proxy/health_check_utils/shared_health_check_manager.py index d0c99d84e94..ae18a42c02b 100644 --- a/litellm/proxy/health_check_utils/shared_health_check_manager.py +++ b/litellm/proxy/health_check_utils/shared_health_check_manager.py @@ -16,7 +16,7 @@ from litellm.proxy.health_check import perform_health_check class SharedHealthCheckManager: """ Manager for coordinating health checks across multiple pods using Redis. - + This class implements a shared health check state mechanism that: - Prevents duplicate health checks across pods - Caches health check results with configurable TTL @@ -58,7 +58,7 @@ class SharedHealthCheckManager: async def acquire_health_check_lock(self) -> bool: """ Attempt to acquire the global health check lock. - + Returns: bool: True if lock was acquired, False otherwise """ @@ -74,7 +74,7 @@ class SharedHealthCheckManager: nx=True, # Only set if key doesn't exist ttl=self.lock_ttl, ) - + if acquired: verbose_proxy_logger.info( "Pod %s acquired health check lock", self.pod_id @@ -83,12 +83,10 @@ class SharedHealthCheckManager: verbose_proxy_logger.debug( "Pod %s failed to acquire health check lock", self.pod_id ) - + return acquired except Exception as e: - verbose_proxy_logger.error( - "Error acquiring health check lock: %s", str(e) - ) + verbose_proxy_logger.error("Error acquiring health check lock: %s", str(e)) return False async def release_health_check_lock(self) -> None: @@ -106,14 +104,12 @@ class SharedHealthCheckManager: "Pod %s released health check lock", self.pod_id ) except Exception as e: - verbose_proxy_logger.error( - "Error releasing health check lock: %s", str(e) - ) + verbose_proxy_logger.error("Error releasing health check lock: %s", str(e)) async def get_cached_health_check_results(self) -> Optional[Dict[str, Any]]: """ Get cached health check results from Redis. - + Returns: Optional[Dict]: Cached health check results or None if not found/expired """ @@ -123,7 +119,7 @@ class SharedHealthCheckManager: try: cache_key = self.get_health_check_cache_key() cached_data = await self.redis_cache.async_get_cache(cache_key) - + if cached_data is None: return None @@ -136,7 +132,7 @@ class SharedHealthCheckManager: # Check if the cache is still valid cache_timestamp = cached_results.get("timestamp", 0) current_time = time.time() - + if current_time - cache_timestamp > self.health_check_ttl: verbose_proxy_logger.debug("Cached health check results expired") return None @@ -151,13 +147,13 @@ class SharedHealthCheckManager: return None async def cache_health_check_results( - self, - healthy_endpoints: List[Dict[str, Any]], - unhealthy_endpoints: List[Dict[str, Any]] + self, + healthy_endpoints: List[Dict[str, Any]], + unhealthy_endpoints: List[Dict[str, Any]], ) -> None: """ Cache health check results in Redis. - + Args: healthy_endpoints: List of healthy endpoints unhealthy_endpoints: List of unhealthy endpoints @@ -181,7 +177,7 @@ class SharedHealthCheckManager: safe_dumps(cache_data), ttl=self.health_check_ttl, ) - + verbose_proxy_logger.info( "Cached health check results for %d healthy and %d unhealthy endpoints", len(healthy_endpoints), @@ -189,29 +185,29 @@ class SharedHealthCheckManager: ) except Exception as e: - verbose_proxy_logger.error( - "Error caching health check results: %s", str(e) - ) + verbose_proxy_logger.error("Error caching health check results: %s", str(e)) async def perform_shared_health_check( - self, - model_list: List[Dict[str, Any]], - details: bool = True + self, + model_list: List[Dict[str, Any]], + details: bool = True, + max_concurrency: Optional[int] = None, ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: """ Perform health check with shared state coordination. - + This method: 1. First checks if there are recent cached results 2. If no recent cache, tries to acquire lock to run health check 3. If lock acquired, runs health check and caches results 4. If lock not acquired, waits briefly and tries to get cached results again 5. Falls back to running health check locally if no cache available - + Args: model_list: List of models to check details: Whether to include detailed information - + max_concurrency: Optional limit on concurrent health check requests + Returns: Tuple of (healthy_endpoints, unhealthy_endpoints) """ @@ -225,27 +221,29 @@ class SharedHealthCheckManager: # No recent cache, try to acquire lock lock_acquired = await self.acquire_health_check_lock() - + if lock_acquired: try: # We have the lock, run health check verbose_proxy_logger.info( - "Pod %s running health check for %d models", - self.pod_id, - len(model_list) + "Pod %s running health check for %d models", + self.pod_id, + len(model_list), ) - + healthy_endpoints, unhealthy_endpoints = await perform_health_check( - model_list=model_list, details=details + model_list=model_list, + details=details, + max_concurrency=max_concurrency, ) - + # Cache the results await self.cache_health_check_results( healthy_endpoints, unhealthy_endpoints ) - + return healthy_endpoints, unhealthy_endpoints - + finally: # Always release the lock await self.release_health_check_lock() @@ -254,10 +252,10 @@ class SharedHealthCheckManager: verbose_proxy_logger.debug( "Pod %s waiting for other pod to complete health check", self.pod_id ) - + # Wait a bit for the other pod to complete await asyncio.sleep(2) - + # Try to get cached results again cached_results = await self.get_cached_health_check_results() if cached_results is not None: @@ -265,19 +263,23 @@ class SharedHealthCheckManager: cached_results.get("healthy_endpoints", []), cached_results.get("unhealthy_endpoints", []), ) - + # Still no cache, fall back to local health check verbose_proxy_logger.warning( - "Pod %s falling back to local health check (no cache available)", - self.pod_id + "Pod %s falling back to local health check (no cache available)", + self.pod_id, + ) + + return await perform_health_check( + model_list=model_list, + details=details, + max_concurrency=max_concurrency, ) - - return await perform_health_check(model_list=model_list, details=details) async def is_health_check_in_progress(self) -> bool: """ Check if a health check is currently in progress by another pod. - + Returns: bool: True if health check is in progress, False otherwise """ @@ -297,7 +299,7 @@ class SharedHealthCheckManager: async def get_health_check_status(self) -> Dict[str, Any]: """ Get the current status of health check coordination. - + Returns: Dict containing status information """ @@ -320,7 +322,9 @@ class SharedHealthCheckManager: cached_results = await self.get_cached_health_check_results() status["cache_available"] = cached_results is not None if cached_results: - status["cache_age_seconds"] = time.time() - cached_results.get("timestamp", 0) + status["cache_age_seconds"] = time.time() - cached_results.get( + "timestamp", 0 + ) status["last_checked_by"] = cached_results.get("checked_by") except Exception as e: diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index da90696ec2d..3570b2dd6aa 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -110,26 +110,31 @@ def _resolve_os_environ_variables(params: dict) -> dict: def get_callback_identifier(callback): """ Get the callback identifier string, handling both strings and objects. - + This function extracts a string identifier from a callback, which can be: - A string (returned as-is) - An object with a callback_name attribute - An object registered in CustomLoggerRegistry - Falls back to callback_name() helper function - + Args: callback: The callback to identify (can be str or object) - + Returns: str: The callback identifier string """ if isinstance(callback, str): return callback - if hasattr(callback, 'callback_name') and callback.callback_name: + if hasattr(callback, "callback_name") and callback.callback_name: return callback.callback_name - if hasattr(callback, '__class__'): - callback_strs = CustomLoggerRegistry.get_all_callback_strs_from_class_type(callback.__class__) - if hasattr(callback, 'callback_name') and callback.callback_name in callback_strs: + if hasattr(callback, "__class__"): + callback_strs = CustomLoggerRegistry.get_all_callback_strs_from_class_type( + callback.__class__ + ) + if ( + hasattr(callback, "callback_name") + and callback.callback_name in callback_strs + ): return callback.callback_name if callback_strs: return callback_strs[0] @@ -151,7 +156,7 @@ services = Union[ "datadog_llm_observability", "generic_api", "arize", - "sqs" + "sqs", ], str, ] @@ -224,7 +229,7 @@ async def health_services_endpoint( # noqa: PLR0915 "datadog_llm_observability", "generic_api", "arize", - "sqs" + "sqs", ]: raise HTTPException( status_code=400, @@ -238,14 +243,14 @@ async def health_services_endpoint( # noqa: PLR0915 service_in_success_callbacks = True else: for cb in litellm.success_callback: - if hasattr(cb, 'callback_name') and cb.callback_name == service: + if hasattr(cb, "callback_name") and cb.callback_name == service: service_in_success_callbacks = True break cb_id = get_callback_identifier(cb) if cb_id == service: service_in_success_callbacks = True break - + if ( service == "openmeter" or service == "braintrust" @@ -320,6 +325,7 @@ async def health_services_endpoint( # noqa: PLR0915 ) elif service == "sqs": from litellm.integrations.sqs import SQSLogger + sqs_logger = SQSLogger() response = await sqs_logger.async_health_check() return { @@ -518,12 +524,12 @@ async def _save_health_check_to_db( def _build_model_param_to_info_mapping(model_list: list) -> dict: """ Build a mapping from model parameter to model info (model_name, model_id). - + Multiple models might share the same model parameter, so we use a list. - + Args: model_list: List of model configurations - + Returns: Dictionary mapping model parameter to list of model info dicts """ @@ -534,14 +540,16 @@ def _build_model_param_to_info_mapping(model_list: list) -> dict: model_id = model_info.get("id") litellm_params = model.get("litellm_params", {}) model_param = litellm_params.get("model") - + if model_param and model_name: if model_param not in model_param_to_info: model_param_to_info[model_param] = [] - model_param_to_info[model_param].append({ - "model_name": model_name, - "model_id": model_id, - }) + model_param_to_info[model_param].append( + { + "model_name": model_name, + "model_id": model_id, + } + ) return model_param_to_info @@ -552,19 +560,19 @@ def _aggregate_health_check_results( ) -> dict: """ Aggregate health check results per unique model. - + Uses (model_id, model_name) as key, or (None, model_name) if model_id is None. - + Args: model_param_to_info: Mapping from model parameter to model info healthy_endpoints: List of healthy endpoint results unhealthy_endpoints: List of unhealthy endpoint results - + Returns: Dictionary mapping (model_id, model_name) to aggregated health check results """ model_results = {} - + # Process healthy endpoints for endpoint in healthy_endpoints: model_param = endpoint.get("model") @@ -580,7 +588,7 @@ def _aggregate_health_check_results( "error_message": None, } model_results[key]["healthy_count"] += 1 - + # Process unhealthy endpoints for endpoint in unhealthy_endpoints: model_param = endpoint.get("model") @@ -600,7 +608,7 @@ def _aggregate_health_check_results( # Use the first error message encountered if not model_results[key]["error_message"] and error_message: model_results[key]["error_message"] = str(error_message)[:500] - + return model_results @@ -613,14 +621,14 @@ async def _save_health_check_results_if_changed( ): """ Save health check results to database, but only if status changed or >1 hour since last save. - + OPTIMIZATION: Only saves to database if the status has changed from the last saved check. This dramatically reduces database writes when health status remains stable. - + - Stable systems: ~1 write/hour per model (instead of 12 writes/hour with 5-min intervals) - Status changes: Immediate write (no delay) - Result: ~92% reduction in DB writes for stable systems, while maintaining real-time updates on changes - + Args: prisma_client: Database client model_results: Dictionary of aggregated health check results per model @@ -630,7 +638,7 @@ async def _save_health_check_results_if_changed( """ for result in model_results.values(): new_status = "healthy" if result["healthy_count"] > 0 else "unhealthy" - + # Check if we should save this result should_save = True lookup_key = result["model_id"] if result["model_id"] else result["model_name"] @@ -641,6 +649,7 @@ async def _save_health_check_results_if_changed( # Check if last check was recent (within 1 hour) if last_check.checked_at: from datetime import datetime, timezone + time_since_last_check = ( datetime.now(timezone.utc) - last_check.checked_at ).total_seconds() @@ -648,7 +657,7 @@ async def _save_health_check_results_if_changed( # This ensures we still get periodic updates even if status is stable if time_since_last_check < 3600: # 1 hour threshold should_save = False - + if should_save: asyncio.create_task( prisma_client.save_health_check_result( @@ -675,27 +684,27 @@ async def _save_background_health_checks_to_db( ): """ Save background health check results to database for each model. - + Maps health check endpoints back to their original models to get model_name and model_id. Aggregates results per unique model (by model_id if available, otherwise model_name). - + OPTIMIZATION: Only saves to database if the status has changed from the last saved check. This dramatically reduces database writes when health status remains stable. """ if prisma_client is None: return - + try: # Step 1: Build mapping from model parameter to model info model_param_to_info = _build_model_param_to_info_mapping(model_list) - + # Step 2: Aggregate health check results per unique model model_results = _aggregate_health_check_results( model_param_to_info, healthy_endpoints, unhealthy_endpoints, ) - + # Step 3: Get latest health checks for all models in one query to compare status latest_checks = await prisma_client.get_all_latest_health_checks() latest_checks_map = {} @@ -704,7 +713,7 @@ async def _save_background_health_checks_to_db( key = check.model_id if check.model_id else check.model_name if key not in latest_checks_map: latest_checks_map[key] = check - + # Step 4: Save aggregated results, but only if status changed await _save_health_check_results_if_changed( prisma_client, @@ -729,10 +738,15 @@ async def _perform_health_check_and_save( start_time, user_id, model_id=None, + max_concurrency=None, ): """Helper function to perform health check and save results to database""" healthy_endpoints, unhealthy_endpoints = await perform_health_check( - model_list=model_list, cli_model=cli_model, model=target_model, details=details + model_list=model_list, + cli_model=cli_model, + model=target_model, + details=details, + max_concurrency=max_concurrency, ) # Optionally save health check result to database (non-blocking) @@ -789,6 +803,7 @@ async def health_endpoint( import time from litellm.proxy.proxy_server import ( + health_check_concurrency, health_check_details, health_check_results, llm_model_list, @@ -841,6 +856,7 @@ async def health_endpoint( start_time=start_time, user_id=user_api_key_dict.user_id, model_id=None, # CLI model doesn't have model_id + max_concurrency=health_check_concurrency, ) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -864,6 +880,7 @@ async def health_endpoint( start_time=start_time, user_id=user_api_key_dict.user_id, model_id=model_id, + max_concurrency=health_check_concurrency, ) except Exception as e: verbose_proxy_logger.error( @@ -1420,11 +1437,11 @@ async def test_model_connection( status_code=500, detail={"error": CommonProxyErrors.db_not_connected_error.value}, ) - + # Get model name from litellm_params request_litellm_params = litellm_params or {} model_name = request_litellm_params.get("model") - + # Look up model configuration from router if model name is provided # This gets the litellm_params from proxy config (with resolved env vars) config_litellm_params: dict = {} @@ -1432,34 +1449,39 @@ async def test_model_connection( try: # First try to find by proxy model_name (e.g., "gpt-4o") deployments = llm_router.get_model_list(model_name=model_name) - + # If not found, try to find by litellm model name (e.g., "azure/gpt-4o") if not deployments or len(deployments) == 0: all_deployments = llm_router.get_model_list(model_name=None) if all_deployments: for deployment in all_deployments: - if deployment.get("litellm_params", {}).get("model") == model_name: + if ( + deployment.get("litellm_params", {}).get("model") + == model_name + ): deployments = [deployment] break - + if deployments and len(deployments) > 0: # Use the first deployment's litellm_params as base config # These already have resolved environment variables from proxy config - config_litellm_params = dict(deployments[0].get("litellm_params", {})) + config_litellm_params = dict( + deployments[0].get("litellm_params", {}) + ) except Exception as e: verbose_proxy_logger.debug( f"Could not find model {model_name} in router: {e}. " "Proceeding with request params only." ) - + # Merge: config params (from proxy config) as base, request params override # This allows users to override specific params while using config for credentials merged_litellm_params = {**config_litellm_params, **request_litellm_params} - + # Resolve os.environ/ environment variables in any remaining request params # This handles cases where user explicitly passes os.environ/ values to override config litellm_params = _resolve_os_environ_variables(merged_litellm_params) - + ## Auth check await ModelManagementAuthChecks.can_user_make_model_call( model_params=Deployment( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 35a75d4caf9..ddab560aaa3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9,6 +9,7 @@ import secrets import shutil import subprocess import sys +import threading import time import traceback import warnings @@ -658,7 +659,7 @@ _description = ( def cleanup_router_config_variables(): - global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, prisma_client + global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, health_check_concurrency, prisma_client # Set all variables to None master_key = None @@ -672,6 +673,7 @@ def cleanup_router_config_variables(): use_background_health_checks = None use_shared_health_check = None health_check_interval = None + health_check_concurrency = None prisma_client = None @@ -822,7 +824,9 @@ async def proxy_startup_event(app: FastAPI): # noqa: PLR0915 verbose_proxy_logger.debug("About to initialize semantic tool filter") _config = proxy_config.get_config_state() _litellm_settings = _config.get("litellm_settings", {}) - verbose_proxy_logger.debug(f"litellm_settings keys = {list(_litellm_settings.keys())}") + verbose_proxy_logger.debug( + f"litellm_settings keys = {list(_litellm_settings.keys())}" + ) await ProxyStartupEvent._initialize_semantic_tool_filter( llm_router=llm_router, litellm_settings=_litellm_settings, @@ -1468,7 +1472,9 @@ redis_usage_cache: Optional[ RedisCache ] = None # redis cache used for tracking spend, tpm/rpm limits polling_via_cache_enabled: Union[Literal["all"], List[str], bool] = False -native_background_mode: List[str] = [] # Models that should use native provider background mode instead of polling +native_background_mode: List[ + str +] = [] # Models that should use native provider background mode instead of polling polling_cache_ttl: int = 3600 # Default 1 hour TTL for polling cache user_custom_auth = None user_custom_key_generate = None @@ -1478,8 +1484,11 @@ use_background_health_checks = None use_shared_health_check = None use_queue = False health_check_interval = None +health_check_concurrency = None health_check_details = None health_check_results: Dict[str, Union[int, List[Dict[str, Any]]]] = {} +background_health_check_loop_active = False +background_health_check_cycle_seq = 0 queue: List = [] litellm_proxy_budget_name = "litellm-proxy-budget" litellm_proxy_admin_name = LITELLM_PROXY_ADMIN_NAME @@ -1927,6 +1936,29 @@ def run_ollama_serve(): ) +def _get_process_rss_mb() -> Optional[float]: + """ + Get process RSS memory in MB. + On Linux, ru_maxrss is in KB. On macOS, ru_maxrss is in bytes. + """ + try: + import resource + + ru_maxrss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + if sys.platform == "darwin": + return float(ru_maxrss) / (1024 * 1024) + return float(ru_maxrss) / 1024 + except Exception: + return None + + +def _rss_mb_for_log() -> str: + rss_mb = _get_process_rss_mb() + if rss_mb is None: + return "unknown" + return f"{rss_mb:.2f}" + + async def _run_background_health_check(): """ Periodically run health checks in the background on the endpoints. @@ -1934,7 +1966,10 @@ async def _run_background_health_check(): Update health_check_results, based on this. Uses shared health check state when Redis is available to coordinate across pods. """ - global health_check_results, llm_model_list, health_check_interval, health_check_details, use_shared_health_check, redis_usage_cache, prisma_client + global health_check_results, llm_model_list, health_check_interval + global health_check_concurrency, health_check_details, use_shared_health_check + global redis_usage_cache, prisma_client + global background_health_check_loop_active, background_health_check_cycle_seq if ( health_check_interval is None @@ -1943,6 +1978,24 @@ async def _run_background_health_check(): ): return + if background_health_check_loop_active: + verbose_proxy_logger.warning( + "background_health_check_loop_overlap_detected existing_loop_active=true interval_seconds=%s max_concurrency=%s shared=%s", + health_check_interval, + health_check_concurrency, + use_shared_health_check, + ) + background_health_check_loop_active = True + verbose_proxy_logger.info( + "background_health_check_loop_started interval_seconds=%s max_concurrency=%s shared=%s details=%s thread_count=%d rss_mb=%s", + health_check_interval, + health_check_concurrency, + use_shared_health_check, + health_check_details, + threading.active_count(), + _rss_mb_for_log(), + ) + # Initialize shared health check manager if Redis is available and feature is enabled shared_health_manager = None if use_shared_health_check and redis_usage_cache is not None: @@ -1958,8 +2011,13 @@ async def _run_background_health_check(): verbose_proxy_logger.info("Initialized shared health check manager") while True: + background_health_check_cycle_seq += 1 + cycle_id = f"bg-{background_health_check_cycle_seq}" + cycle_start_time = time.monotonic() + # make 1 deep copy of llm_model_list on every health check iteration _llm_model_list = copy.deepcopy(llm_model_list) or [] + model_count_total = len(_llm_model_list) # filter out models that have disabled background health checks _llm_model_list = [ @@ -1967,6 +2025,52 @@ async def _run_background_health_check(): for m in _llm_model_list if not m.get("model_info", {}).get("disable_background_health_check", False) ] + model_count_enabled = len(_llm_model_list) + expected_peak_in_flight = model_count_enabled + if ( + isinstance(health_check_concurrency, int) + and health_check_concurrency > 0 + and model_count_enabled > 0 + ): + expected_peak_in_flight = min(model_count_enabled, health_check_concurrency) + + verbose_proxy_logger.debug( + "background_health_check_cycle_start cycle_id=%s model_count_total=%d model_count_enabled=%d interval_seconds=%s max_concurrency=%s expected_peak_in_flight=%d shared=%s thread_count=%d rss_mb=%s", + cycle_id, + model_count_total, + model_count_enabled, + health_check_interval, + health_check_concurrency, + expected_peak_in_flight, + shared_health_manager is not None, + threading.active_count(), + _rss_mb_for_log(), + ) + + instrumentation_context = { + "enabled": True, + "source": "proxy_background_loop", + "cycle_id": cycle_id, + } + + async def _run_direct_health_check_with_instrumentation(): + try: + return await perform_health_check( + model_list=_llm_model_list, + details=health_check_details, + max_concurrency=health_check_concurrency, + instrumentation_context=instrumentation_context, + ) + except TypeError as e: + if "instrumentation_context" not in str(e): + raise + # Backward compatibility for monkeypatched or wrapped callables + # that do not accept instrumentation_context. + return await perform_health_check( + model_list=_llm_model_list, + details=health_check_details, + max_concurrency=health_check_concurrency, + ) # Use shared health check if available, otherwise fall back to direct health check # Convert health_check_details to bool for perform_shared_health_check (defaults to True if None) @@ -1980,19 +2084,21 @@ async def _run_background_health_check(): healthy_endpoints, unhealthy_endpoints, ) = await shared_health_manager.perform_shared_health_check( - model_list=_llm_model_list, details=details_bool + model_list=_llm_model_list, + details=details_bool, + max_concurrency=health_check_concurrency, ) except Exception as e: verbose_proxy_logger.error( "Error in shared health check, falling back to direct health check: %s", str(e), ) - healthy_endpoints, unhealthy_endpoints = await perform_health_check( - model_list=_llm_model_list, details=health_check_details + healthy_endpoints, unhealthy_endpoints = ( + await _run_direct_health_check_with_instrumentation() ) else: - healthy_endpoints, unhealthy_endpoints = await perform_health_check( - model_list=_llm_model_list, details=health_check_details + healthy_endpoints, unhealthy_endpoints = ( + await _run_direct_health_check_with_instrumentation() ) # Update the global variable with the health check results @@ -2000,6 +2106,25 @@ async def _run_background_health_check(): health_check_results["unhealthy_endpoints"] = unhealthy_endpoints health_check_results["healthy_count"] = len(healthy_endpoints) health_check_results["unhealthy_count"] = len(unhealthy_endpoints) + cycle_duration_ms = (time.monotonic() - cycle_start_time) * 1000 + verbose_proxy_logger.debug( + "background_health_check_cycle_complete cycle_id=%s model_count_enabled=%d healthy_count=%d unhealthy_count=%d duration_ms=%.2f interval_seconds=%s thread_count=%d rss_mb=%s", + cycle_id, + model_count_enabled, + len(healthy_endpoints), + len(unhealthy_endpoints), + cycle_duration_ms, + health_check_interval, + threading.active_count(), + _rss_mb_for_log(), + ) + if cycle_duration_ms > (health_check_interval * 1000): + verbose_proxy_logger.warning( + "background_health_check_cycle_duration_exceeded_interval cycle_id=%s duration_ms=%.2f interval_seconds=%s", + cycle_id, + cycle_duration_ms, + health_check_interval, + ) # Save background health checks to database (non-blocking) if prisma_client is not None: @@ -2480,7 +2605,7 @@ class ProxyConfig: """ Load config values into proxy global state """ - global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, proxy_batch_polling_interval, config_passthrough_endpoints + global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, health_check_concurrency, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, proxy_batch_polling_interval, config_passthrough_endpoints config: dict = await self.get_config(config_file_path=config_file_path) @@ -2905,7 +3030,18 @@ class ProxyConfig: health_check_interval = general_settings.get( "health_check_interval", DEFAULT_HEALTH_CHECK_INTERVAL ) + health_check_concurrency = general_settings.get( + "health_check_concurrency", None + ) health_check_details = general_settings.get("health_check_details", True) + verbose_proxy_logger.info( + "background_health_check_config enabled=%s shared=%s interval_seconds=%s max_concurrency=%s details=%s", + use_background_health_checks, + use_shared_health_check, + health_check_interval, + health_check_concurrency, + health_check_details, + ) ### RBAC ### rbac_role_permissions = general_settings.get("role_permissions", None) @@ -2999,7 +3135,7 @@ class ProxyConfig: for k, v in router_settings.items(): if k in available_args: router_params[k] = v - elif k == "health_check_interval": + elif k in {"health_check_interval", "health_check_concurrency"}: raise ValueError( f"'{k}' is NOT a valid router_settings parameter. Please move it to 'general_settings'." ) @@ -4201,9 +4337,7 @@ class ProxyConfig: ) if self._should_load_db_object(object_type="semantic_filter_settings"): - await self._init_semantic_filter_settings_in_db( - prisma_client=prisma_client - ) + await self._init_semantic_filter_settings_in_db(prisma_client=prisma_client) async def _init_semantic_filter_settings_in_db(self, prisma_client: PrismaClient): """ @@ -5259,30 +5393,38 @@ class ProxyStartupEvent: ): """Initialize MCP semantic tool filter if configured""" from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook - - mcp_semantic_filter_config = litellm_settings.get("mcp_semantic_tool_filter", None) - + + mcp_semantic_filter_config = litellm_settings.get( + "mcp_semantic_tool_filter", None + ) + # Only proceed if the feature is configured and enabled - if not mcp_semantic_filter_config or not mcp_semantic_filter_config.get("enabled", False): - verbose_proxy_logger.debug("Semantic tool filter not configured or not enabled, skipping initialization") + if not mcp_semantic_filter_config or not mcp_semantic_filter_config.get( + "enabled", False + ): + verbose_proxy_logger.debug( + "Semantic tool filter not configured or not enabled, " + "skipping initialization" + ) return - + verbose_proxy_logger.debug( f"Initializing semantic tool filter: llm_router={llm_router is not None}, " f"config={mcp_semantic_filter_config}" ) - hook = await SemanticToolFilterHook.initialize_from_config( config=mcp_semantic_filter_config, llm_router=llm_router, ) - + if hook: verbose_proxy_logger.debug("Semantic tool filter hook registered") litellm.logging_callback_manager.add_litellm_callback(hook) else: # Only warn if the feature was configured but failed to initialize - verbose_proxy_logger.warning("Semantic tool filter hook was configured but failed to initialize") + verbose_proxy_logger.warning( + "Semantic tool filter hook was configured but failed to initialize" + ) @classmethod def _initialize_jwt_auth( @@ -8706,7 +8848,8 @@ async def _apply_search_filter_to_models( # Fetch database models if we need more for the current page if router_models_count < models_needed_for_page: models_to_fetch = min( - models_needed_for_page - router_models_count, db_models_total_count + models_needed_for_page - router_models_count, + db_models_total_count, ) if models_to_fetch > 0: @@ -8742,21 +8885,21 @@ async def _apply_search_filter_to_models( def _normalize_datetime_for_sorting(dt: Any) -> Optional[datetime]: """ Normalize a datetime value to a timezone-aware UTC datetime for sorting. - + This function handles: - None values: returns None - String values: parses ISO format strings and converts to UTC-aware datetime - Datetime objects: converts naive datetimes to UTC-aware, and aware datetimes to UTC - + Args: dt: Datetime value (None, str, or datetime object) - + Returns: UTC-aware datetime object, or None if input is None or cannot be parsed """ if dt is None: return None - + if isinstance(dt, str): try: # Handle ISO format strings, including 'Z' suffix @@ -8770,14 +8913,14 @@ def _normalize_datetime_for_sorting(dt: Any) -> Optional[datetime]: return parsed_dt except (ValueError, AttributeError): return None - + if isinstance(dt, datetime): # If naive, assume UTC and make it aware if dt.tzinfo is None: return dt.replace(tzinfo=timezone.utc) # If aware, convert to UTC return dt.astimezone(timezone.utc) - + return None @@ -8797,46 +8940,60 @@ def _sort_models( Returns: Sorted list of models """ - if not sort_by or sort_by not in ["model_name", "created_at", "updated_at", "costs", "status"]: + if not sort_by or sort_by not in [ + "model_name", + "created_at", + "updated_at", + "costs", + "status", + ]: return all_models reverse = sort_order.lower() == "desc" def get_sort_key(model: Dict[str, Any]) -> Any: model_info = model.get("model_info", {}) - + if sort_by == "model_name": return model.get("model_name", "").lower() - + elif sort_by == "created_at": created_at = model_info.get("created_at") normalized_dt = _normalize_datetime_for_sorting(created_at) if normalized_dt is None: # Put None values at the end for asc, at the start for desc - return (datetime.max.replace(tzinfo=timezone.utc) if not reverse else datetime.min.replace(tzinfo=timezone.utc)) + return ( + datetime.max.replace(tzinfo=timezone.utc) + if not reverse + else datetime.min.replace(tzinfo=timezone.utc) + ) return normalized_dt - + elif sort_by == "updated_at": updated_at = model_info.get("updated_at") normalized_dt = _normalize_datetime_for_sorting(updated_at) if normalized_dt is None: - return (datetime.max.replace(tzinfo=timezone.utc) if not reverse else datetime.min.replace(tzinfo=timezone.utc)) + return ( + datetime.max.replace(tzinfo=timezone.utc) + if not reverse + else datetime.min.replace(tzinfo=timezone.utc) + ) return normalized_dt - + elif sort_by == "costs": input_cost = model_info.get("input_cost_per_token", 0) or 0 output_cost = model_info.get("output_cost_per_token", 0) or 0 total_cost = input_cost + output_cost # Put 0 or None costs at the end for asc, at the start for desc if total_cost == 0: - return (float("inf") if not reverse else float("-inf")) + return float("inf") if not reverse else float("-inf") return total_cost - + elif sort_by == "status": # False (config) comes before True (db) for asc db_model = model_info.get("db_model", False) return db_model - + return None try: @@ -9032,9 +9189,7 @@ async def _find_model_by_id( ) if db_model: # Convert database model to router format - decrypted_models = proxy_config.decrypt_model_list_from_db( - [db_model] - ) + decrypted_models = proxy_config.decrypt_model_list_from_db([db_model]) if decrypted_models: found_model = decrypted_models[0] except Exception as e: @@ -9208,13 +9363,13 @@ async def model_info_v2( ) verbose_proxy_logger.debug("all_models: %s", all_models) - + # Append A2A agents to models list all_models = await append_agents_to_model_info( models=all_models, user_api_key_dict=user_api_key_dict, ) - + # Update total count to include agents search_total_count = len(all_models) @@ -10057,7 +10212,7 @@ async def model_group_info( model_groups: List[ModelGroupInfoProxy] = _get_model_group_info( llm_router=llm_router, all_models_str=all_models_str, model_group=model_group ) - + # Append A2A agents to model groups model_groups = await append_agents_to_model_group( model_groups=model_groups, diff --git a/tests/litellm_utils_tests/test_health_check.py b/tests/litellm_utils_tests/test_health_check.py index 19882bbe4be..7ea4574bab9 100644 --- a/tests/litellm_utils_tests/test_health_check.py +++ b/tests/litellm_utils_tests/test_health_check.py @@ -92,7 +92,7 @@ async def test_azure_img_gen_health_check(): litellm._turn_on_debug() max_retries = 3 retry_delay = 1 # Start with 1 second delay - + for attempt in range(max_retries): response = await litellm.ahealth_check( model_params={ @@ -103,11 +103,11 @@ async def test_azure_img_gen_health_check(): mode="image_generation", prompt="cute baby sea otter", ) - + # Check if response is successful (no error) if isinstance(response, dict) and "error" not in response: return response - + # Check if error is a transient Azure internal server error error_str = str(response.get("error", "")).lower() is_transient_error = ( @@ -116,16 +116,18 @@ async def test_azure_img_gen_health_check(): or "internalfailure" in error_str or "internal failure" in error_str ) - + # If it's the last attempt or not a transient error, fail the test if attempt == max_retries - 1 or not is_transient_error: - assert isinstance(response, dict) and "error" not in response, f"Health check failed: {response.get('error', 'Unknown error')}" + assert ( + isinstance(response, dict) and "error" not in response + ), f"Health check failed: {response.get('error', 'Unknown error')}" return response - + # Wait before retrying with exponential backoff await asyncio.sleep(retry_delay) retry_delay *= 2 # Exponential backoff - + # Should not reach here, but just in case assert False, "Health check failed after all retries" @@ -562,6 +564,99 @@ async def test_health_check_bad_model(): ), "Health check took longer than health_check_timeout" +@pytest.mark.asyncio +async def test_health_check_respects_concurrency_limit(): + from litellm.proxy.health_check import _perform_health_check + + model_list = [ + {"litellm_params": {"model": f"openai/gpt-4o-mini-{i}", "api_key": "fake-key"}} + for i in range(6) + ] + + active = 0 + max_active = 0 + + async def mock_health_check(litellm_params, **kwargs): + nonlocal active, max_active + active += 1 + max_active = max(max_active, active) + await asyncio.sleep(0.05) + active -= 1 + return {"status": "healthy"} + + with patch("litellm.ahealth_check", side_effect=mock_health_check): + await _perform_health_check(model_list, max_concurrency=2) + + assert max_active <= 2 + + +@pytest.mark.asyncio +async def test_health_check_creates_only_bounded_initial_tasks(): + from litellm.proxy.health_check import _perform_health_check + + model_list = [ + {"litellm_params": {"model": f"openai/gpt-4o-mini-{i}", "api_key": "fake-key"}} + for i in range(10) + ] + release_event = asyncio.Event() + create_task_call_count = 0 + real_create_task = asyncio.create_task + + async def mock_health_check(litellm_params, **kwargs): + await release_event.wait() + return {"status": "healthy"} + + def tracked_create_task(coro): + nonlocal create_task_call_count + create_task_call_count += 1 + return real_create_task(coro) + + with patch("litellm.ahealth_check", side_effect=mock_health_check), patch( + "litellm.proxy.health_check.asyncio.create_task", side_effect=tracked_create_task + ): + perform_task = real_create_task( + _perform_health_check(model_list, max_concurrency=2) + ) + await asyncio.sleep(0.05) + assert create_task_call_count == 2 + release_event.set() + await perform_task + + +@pytest.mark.asyncio +async def test_timeout_does_not_cancel_other_health_checks(): + from litellm.proxy.health_check import _perform_health_check + + model_list = [ + { + "litellm_params": {"model": "openai/slow-model", "api_key": "fake-key"}, + "model_info": {"health_check_timeout": 0.05}, + }, + { + "litellm_params": {"model": "openai/fast-model", "api_key": "fake-key"}, + "model_info": {"health_check_timeout": 1}, + }, + ] + + async def mock_health_check(litellm_params, **kwargs): + if litellm_params["model"] == "openai/slow-model": + await asyncio.sleep(0.2) + return {"status": "healthy"} + await asyncio.sleep(0.01) + return {"status": "healthy"} + + with patch("litellm.ahealth_check", side_effect=mock_health_check): + healthy_endpoints, unhealthy_endpoints = await _perform_health_check( + model_list, max_concurrency=1 + ) + + healthy_models = {endpoint["model"] for endpoint in healthy_endpoints} + unhealthy_models = {endpoint["model"] for endpoint in unhealthy_endpoints} + + assert "openai/fast-model" in healthy_models + assert "openai/slow-model" in unhealthy_models + + @pytest.mark.asyncio async def test_ahealth_check_ocr(): litellm._turn_on_debug() @@ -643,20 +738,20 @@ async def test_image_generation_health_check_prompt(monkeypatch): async def test_health_check_with_custom_llm_provider(): """ Test that ahealth_check correctly uses custom_llm_provider from model_params. - + This test verifies the fix for the issue where the UI's "Test connect" button failed with "LLM Provider NOT provided" error for OpenAI-compatible self-hosted providers, even when a provider was selected in the dropdown. - + The fix ensures that when custom_llm_provider is passed in model_params, it's properly forwarded to get_llm_provider() to identify the correct provider. """ from unittest.mock import MagicMock - + # Mock the completion call to avoid making real API calls mock_response = MagicMock() mock_response._hidden_params = {"headers": {"x-ratelimit-remaining-tokens": "1000"}} - + with patch("litellm.acompletion", return_value=mock_response): # Test with a custom model name that wouldn't be recognized without custom_llm_provider response = await litellm.ahealth_check( @@ -668,7 +763,7 @@ async def test_health_check_with_custom_llm_provider(): }, mode="chat", ) - + # Should succeed without "LLM Provider NOT provided" error assert "error" not in response assert isinstance(response, dict) diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index 14b49901c9c..0cbba7b5cc3 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -2330,7 +2330,9 @@ async def test_run_background_health_check_reflects_llm_model_list(monkeypatch): test_model_list_2 = [{"model_name": "model-b"}] called_model_lists = [] - async def fake_perform_health_check(model_list, details): + async def fake_perform_health_check( + model_list, details, max_concurrency=None + ): called_model_lists.append(copy.deepcopy(model_list)) return (["healthy"], ["unhealthy"]) @@ -2378,7 +2380,9 @@ async def test_background_health_check_skip_disabled_models(monkeypatch): ] called_model_lists = [] - async def fake_perform_health_check(model_list, details): + async def fake_perform_health_check( + model_list, details, max_concurrency=None + ): called_model_lists.append(copy.deepcopy(model_list)) return (["healthy"], []) From 5e9f24f74c59d0272b050a17d0b34ae82646e9fb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 24 Feb 2026 09:32:11 -0800 Subject: [PATCH 06/12] fix(bedrock): pass timeout param to bedrock rerank http client (#22021) * fix(bedrock): pass timeout to bedrock rerank http client * refactor: extract large functions to fix PLR0915 ruff lint errors --- .../litellm_core_utils/realtime_streaming.py | 248 +++++++++--------- litellm/llms/bedrock/rerank/handler.py | 8 +- ...odel_prices_and_context_window_backup.json | 95 +++++-- litellm/proxy/health_check.py | 122 ++++----- litellm/proxy/proxy_server.py | 127 +++++---- litellm/rerank_api/main.py | 1 + .../test_bedrock_rerank_header_forwarding.py | 77 ++++++ 7 files changed, 427 insertions(+), 251 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index b42bf9e5711..759eaf60035 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -203,6 +203,129 @@ class RealTimeStreaming: return True return False + async def _handle_provider_config_message(self, raw_response) -> None: + """Process a backend message when a provider_config is set (transformed path).""" + returned_object = self.provider_config.transform_realtime_response( # type: ignore[union-attr] + raw_response, + self.model, + self.logging_obj, + realtime_response_transform_input={ + "session_configuration_request": self.session_configuration_request, + "current_output_item_id": self.current_output_item_id, + "current_response_id": self.current_response_id, + "current_delta_chunks": self.current_delta_chunks, + "current_conversation_id": self.current_conversation_id, + "current_item_chunks": self.current_item_chunks, + "current_delta_type": self.current_delta_type, + }, + ) + + transformed_response = returned_object["response"] + self.current_output_item_id = returned_object["current_output_item_id"] + self.current_response_id = returned_object["current_response_id"] + self.current_delta_chunks = returned_object["current_delta_chunks"] + self.current_conversation_id = returned_object["current_conversation_id"] + self.current_item_chunks = returned_object["current_item_chunks"] + self.current_delta_type = returned_object["current_delta_type"] + self.session_configuration_request = returned_object["session_configuration_request"] + events = ( + transformed_response + if isinstance(transformed_response, list) + else [transformed_response] + ) + for event in events: + ## GUARDRAIL: inject create_response=false on session.created + if isinstance(event, dict) and event.get("type") == "session.created": + if self._has_realtime_guardrails(): + await self.backend_ws.send( + json.dumps( + { + "type": "session.update", + "session": { + "turn_detection": { + "type": "server_vad", + "create_response": False, + } + }, + } + ) + ) + for event in events: + event_str = json.dumps(event) + ## GUARDRAIL: run on transcription events in provider_config path too + if ( + isinstance(event, dict) + and event.get("type") + == "conversation.item.input_audio_transcription.completed" + ): + transcript = event.get("transcript", "") + self.store_message(event_str) + await self.websocket.send_text(event_str) + blocked = await self.run_realtime_guardrails( + transcript, item_id=event.get("item_id") + ) + if not blocked: + await self.backend_ws.send( + json.dumps({"type": "response.create"}) + ) + continue + ## LOGGING + self.store_message(event_str) + await self.websocket.send_text(event_str) + + async def _handle_raw_backend_message(self, raw_response) -> bool: + """Process a backend message without provider_config (raw path). + + Returns True if the caller should skip the default store+forward (i.e. continue the loop). + """ + try: + event_obj = json.loads(raw_response) + + if event_obj.get("type") == "session.created": + # If any realtime guardrails are registered, proactively + # set create_response=false so the LLM never auto-responds + # before our guardrail has a chance to run. + if self._has_realtime_guardrails(): + await self.backend_ws.send( + json.dumps( + { + "type": "session.update", + "session": { + "turn_detection": { + "type": "server_vad", + "create_response": False, + } + }, + } + ) + ) + verbose_logger.debug( + "[realtime guardrail] injected create_response=false into session" + ) + + if ( + event_obj.get("type") + == "conversation.item.input_audio_transcription.completed" + ): + transcript = event_obj.get("transcript", "") + ## LOGGING — must happen before continue below + self.store_message(raw_response) + # Forward transcript to client so user sees what they said + await self.websocket.send_text(raw_response) + blocked = await self.run_realtime_guardrails( + transcript, + item_id=event_obj.get("item_id"), + ) + if not blocked: + # Clean — trigger LLM response + await self.backend_ws.send( + json.dumps({"type": "response.create"}) + ) + return True + except (json.JSONDecodeError, AttributeError): + pass + return False + async def backend_to_client_send_messages(self): import websockets @@ -216,128 +339,11 @@ class RealTimeStreaming: raw_response = await self.backend_ws.recv() # type: ignore[assignment] if self.provider_config: - returned_object = self.provider_config.transform_realtime_response( - raw_response, - self.model, - self.logging_obj, - realtime_response_transform_input={ - "session_configuration_request": self.session_configuration_request, - "current_output_item_id": self.current_output_item_id, - "current_response_id": self.current_response_id, - "current_delta_chunks": self.current_delta_chunks, - "current_conversation_id": self.current_conversation_id, - "current_item_chunks": self.current_item_chunks, - "current_delta_type": self.current_delta_type, - }, - ) - - transformed_response = returned_object["response"] - self.current_output_item_id = returned_object[ - "current_output_item_id" - ] - self.current_response_id = returned_object["current_response_id"] - self.current_delta_chunks = returned_object["current_delta_chunks"] - self.current_conversation_id = returned_object[ - "current_conversation_id" - ] - self.current_item_chunks = returned_object["current_item_chunks"] - self.current_delta_type = returned_object["current_delta_type"] - self.session_configuration_request = returned_object[ - "session_configuration_request" - ] - events = ( - transformed_response - if isinstance(transformed_response, list) - else [transformed_response] - ) - for event in events: - ## GUARDRAIL: inject create_response=false on session.created - if isinstance(event, dict) and event.get("type") == "session.created": - if self._has_realtime_guardrails(): - await self.backend_ws.send( - json.dumps( - { - "type": "session.update", - "session": { - "turn_detection": { - "type": "server_vad", - "create_response": False, - } - }, - } - ) - ) - for event in events: - event_str = json.dumps(event) - ## GUARDRAIL: run on transcription events in provider_config path too - if ( - isinstance(event, dict) - and event.get("type") - == "conversation.item.input_audio_transcription.completed" - ): - transcript = event.get("transcript", "") - self.store_message(event_str) - await self.websocket.send_text(event_str) - blocked = await self.run_realtime_guardrails( - transcript, item_id=event.get("item_id") - ) - if not blocked: - await self.backend_ws.send( - json.dumps({"type": "response.create"}) - ) - continue - ## LOGGING - self.store_message(event_str) - await self.websocket.send_text(event_str) - + await self._handle_provider_config_message(raw_response) else: - ## GUARDRAIL: intercept transcription events before triggering LLM - try: - event_obj = json.loads(raw_response) - - if event_obj.get("type") == "session.created": - # If any realtime guardrails are registered, proactively - # set create_response=false so the LLM never auto-responds - # before our guardrail has a chance to run. - if self._has_realtime_guardrails(): - await self.backend_ws.send( - json.dumps( - { - "type": "session.update", - "session": { - "turn_detection": { - "type": "server_vad", - "create_response": False, - } - }, - } - ) - ) - verbose_logger.debug( - "[realtime guardrail] injected create_response=false into session" - ) - - if ( - event_obj.get("type") - == "conversation.item.input_audio_transcription.completed" - ): - transcript = event_obj.get("transcript", "") - ## LOGGING — must happen before continue below - self.store_message(raw_response) - # Forward transcript to client so user sees what they said - await self.websocket.send_text(raw_response) - blocked = await self.run_realtime_guardrails( - transcript, - item_id=event_obj.get("item_id"), - ) - if not blocked: - # Clean — trigger LLM response - await self.backend_ws.send( - json.dumps({"type": "response.create"}) - ) - continue - except (json.JSONDecodeError, AttributeError): - pass + handled = await self._handle_raw_backend_message(raw_response) + if handled: + continue ## LOGGING self.store_message(raw_response) await self.websocket.send_text(raw_response) diff --git a/litellm/llms/bedrock/rerank/handler.py b/litellm/llms/bedrock/rerank/handler.py index 06f1e9e86c9..37167e7c330 100644 --- a/litellm/llms/bedrock/rerank/handler.py +++ b/litellm/llms/bedrock/rerank/handler.py @@ -29,12 +29,13 @@ class BedrockRerankHandler(BaseAWSLLM): async def arerank( self, prepared_request: BedrockPreparedRequest, + timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[AsyncHTTPHandler] = None, ): if client is None: client = get_async_httpx_client(llm_provider=litellm.LlmProviders.BEDROCK) try: - response = await client.post(url=prepared_request["endpoint_url"], headers=dict(prepared_request["prepped"].headers), data=prepared_request["body"]) + response = await client.post(url=prepared_request["endpoint_url"], headers=dict(prepared_request["prepped"].headers), data=prepared_request["body"], timeout=timeout) response.raise_for_status() except httpx.HTTPStatusError as err: error_code = err.response.status_code @@ -56,6 +57,7 @@ class BedrockRerankHandler(BaseAWSLLM): return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, _is_async: Optional[bool] = False, + timeout: Optional[Union[float, httpx.Timeout]] = None, api_base: Optional[str] = None, extra_headers: Optional[dict] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, @@ -89,12 +91,12 @@ class BedrockRerankHandler(BaseAWSLLM): ) if _is_async: - return self.arerank(prepared_request, client=client if client is not None and isinstance(client, AsyncHTTPHandler) else None) # type: ignore + return self.arerank(prepared_request, timeout=timeout, client=client if client is not None and isinstance(client, AsyncHTTPHandler) else None) # type: ignore if client is None or not isinstance(client, HTTPHandler): client = _get_httpx_client() try: - response = client.post(url=prepared_request["endpoint_url"], headers=dict(prepared_request["prepped"].headers), data=prepared_request["body"]) + response = client.post(url=prepared_request["endpoint_url"], headers=dict(prepared_request["prepped"].headers), data=prepared_request["body"], timeout=timeout) response.raise_for_status() except httpx.HTTPStatusError as err: error_code = err.response.status_code diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3909ce4c8b0..624714feb87 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -26555,65 +26555,124 @@ "supports_function_calling": true, "supports_tool_choice": true }, + "perplexity/preset/fast-search": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_preset": true, + "supports_function_calling": true + }, "perplexity/preset/pro-search": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_preset": true + "supports_preset": true, + "supports_function_calling": true }, - "perplexity/openai/gpt-4o": { + "perplexity/preset/deep-research": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": false + "supports_preset": true, + "supports_function_calling": true }, - "perplexity/openai/gpt-4o-mini": { + "perplexity/preset/advanced-deep-research": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": false + "supports_preset": true, + "supports_function_calling": true }, "perplexity/openai/gpt-5.2": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_function_calling": true }, - "perplexity/anthropic/claude-3-5-sonnet-20241022": { + "perplexity/openai/gpt-5.1": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": false + "supports_reasoning": false, + "supports_function_calling": true }, - "perplexity/anthropic/claude-3-5-haiku-20241022": { + "perplexity/openai/gpt-5-mini": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": false + "supports_reasoning": false, + "supports_function_calling": true }, - "perplexity/google/gemini-2.0-flash-exp": { + "perplexity/anthropic/claude-opus-4-6": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": false + "supports_reasoning": false, + "supports_function_calling": true }, - "perplexity/google/gemini-2.0-flash-thinking-exp": { + "perplexity/anthropic/claude-opus-4-5": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": true + "supports_reasoning": false, + "supports_function_calling": true }, - "perplexity/xai/grok-2-1212": { + "perplexity/anthropic/claude-sonnet-4-5": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": false + "supports_reasoning": false, + "supports_function_calling": true }, - "perplexity/xai/grok-2-vision-1212": { + "perplexity/anthropic/claude-haiku-4-5": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": false + "supports_reasoning": false, + "supports_function_calling": true + }, + "perplexity/google/gemini-3-pro-preview": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false, + "supports_function_calling": true + }, + "perplexity/google/gemini-3-flash-preview": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false, + "supports_function_calling": true + }, + "perplexity/google/gemini-2.5-pro": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false, + "supports_function_calling": true + }, + "perplexity/google/gemini-2.5-flash": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false, + "supports_function_calling": true + }, + "perplexity/xai/grok-4-1-fast-non-reasoning": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false, + "supports_function_calling": true + }, + "perplexity/perplexity/sonar": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false, + "supports_function_calling": true }, "publicai/aisingapore/Qwen-SEA-LION-v4-32B-IT": { "input_cost_per_token": 0.0, diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 4777f64405d..de4517efc88 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -11,7 +11,7 @@ from typing import List, Optional import litellm logger = logging.getLogger(__name__) -from litellm.constants import HEALTH_CHECK_TIMEOUT_SECONDS, DEFAULT_HEALTH_CHECK_PROMPT +from litellm.constants import DEFAULT_HEALTH_CHECK_PROMPT, HEALTH_CHECK_TIMEOUT_SECONDS ILLEGAL_DISPLAY_PARAMS = [ "messages", @@ -98,6 +98,66 @@ async def run_with_timeout(task, timeout): return {"error": "Timeout exceeded"} +async def _run_model_health_check(model: dict): + litellm_params = model["litellm_params"] + model_info = model.get("model_info", {}) + mode = model_info.get("mode", None) + litellm_params = _update_litellm_params_for_health_check(model_info, litellm_params) + timeout = model_info.get("health_check_timeout") or HEALTH_CHECK_TIMEOUT_SECONDS + + return await run_with_timeout( + litellm.ahealth_check( + litellm_params, + mode=mode, + prompt=DEFAULT_HEALTH_CHECK_PROMPT, + input=["test from litellm"], + ), + timeout, + ) + + +async def _run_health_checks_with_bounded_concurrency( + models: list, concurrency_limit: int +) -> tuple[list, int]: + """ + Run health checks with at most `concurrency_limit` active tasks. + Preserves result ordering to match `models`. + """ + results: list = [None] * len(models) + tasks_to_index: dict[asyncio.Task, int] = {} + model_iter = iter(enumerate(models)) + peak_in_flight = 0 + + def _schedule_next() -> bool: + nonlocal peak_in_flight + try: + idx, next_model = next(model_iter) + except StopIteration: + return False + task = asyncio.create_task(_run_model_health_check(next_model)) + tasks_to_index[task] = idx + peak_in_flight = max(peak_in_flight, len(tasks_to_index)) + return True + + for _ in range(min(concurrency_limit, len(models))): + _schedule_next() + + while tasks_to_index: + done, _ = await asyncio.wait( + set(tasks_to_index.keys()), + return_when=asyncio.FIRST_COMPLETED, + ) + for task in done: + idx = tasks_to_index.pop(task) + try: + results[idx] = task.result() + except Exception as e: + results[idx] = e + _schedule_next() + + return results, peak_in_flight + + async def _perform_health_check( model_list: list, details: Optional[bool] = True, @@ -115,66 +175,6 @@ async def _perform_health_check( cycle_id = instrumentation_context.get("cycle_id", "unknown") source = instrumentation_context.get("source", "unknown") - async def _run_model_health_check(model: dict): - litellm_params = model["litellm_params"] - model_info = model.get("model_info", {}) - mode = model_info.get("mode", None) - litellm_params = _update_litellm_params_for_health_check( - model_info, litellm_params - ) - timeout = model_info.get("health_check_timeout") or HEALTH_CHECK_TIMEOUT_SECONDS - - return await run_with_timeout( - litellm.ahealth_check( - litellm_params, - mode=mode, - prompt=DEFAULT_HEALTH_CHECK_PROMPT, - input=["test from litellm"], - ), - timeout, - ) - - async def _run_health_checks_with_bounded_concurrency( - models: list, concurrency_limit: int - ) -> tuple[list, int]: - """ - Run health checks with at most `concurrency_limit` active tasks. - Preserves result ordering to match `models`. - """ - results: list = [None] * len(models) - tasks_to_index: dict[asyncio.Task, int] = {} - model_iter = iter(enumerate(models)) - peak_in_flight = 0 - - def _schedule_next() -> bool: - nonlocal peak_in_flight - try: - idx, next_model = next(model_iter) - except StopIteration: - return False - task = asyncio.create_task(_run_model_health_check(next_model)) - tasks_to_index[task] = idx - peak_in_flight = max(peak_in_flight, len(tasks_to_index)) - return True - - for _ in range(min(concurrency_limit, len(models))): - _schedule_next() - - while tasks_to_index: - done, _ = await asyncio.wait( - set(tasks_to_index.keys()), - return_when=asyncio.FIRST_COMPLETED, - ) - for task in done: - idx = tasks_to_index.pop(task) - try: - results[idx] = task.result() - except Exception as e: - results[idx] = e - _schedule_next() - - return results, peak_in_flight - dispatch_mode = "unbounded" peak_in_flight = 0 if isinstance(max_concurrency, int) and max_concurrency > 0: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ddab560aaa3..b3b6bf0ccf7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -713,7 +713,7 @@ async def _initialize_shared_aiohttp_session(): try: from aiohttp import ClientSession, TCPConnector - connector_kwargs = { + connector_kwargs: Dict[str, Any] = { "keepalive_timeout": AIOHTTP_KEEPALIVE_TIMEOUT, "ttl_dns_cache": AIOHTTP_TTL_DNS_CACHE, } @@ -1959,6 +1959,65 @@ def _rss_mb_for_log() -> str: return f"{rss_mb:.2f}" +async def _run_direct_health_check_with_instrumentation( + model_list: list, + details: Optional[bool], + max_concurrency: Optional[int], + instrumentation_context: dict, +): + try: + return await perform_health_check( + model_list=model_list, + details=details, + max_concurrency=max_concurrency, + instrumentation_context=instrumentation_context, + ) + except TypeError as e: + if "instrumentation_context" not in str(e): + raise + # Backward compatibility for monkeypatched or wrapped callables + # that do not accept instrumentation_context. + return await perform_health_check( + model_list=model_list, + details=details, + max_concurrency=max_concurrency, + ) + + +def _schedule_background_health_check_db_save( + prisma_client, + shared_health_manager, + model_list: list, + healthy_endpoints: list, + unhealthy_endpoints: list, +): + """Fire-and-forget: persist health check results to DB if prisma is available.""" + if prisma_client is None: + return + import time as time_module + + from litellm.proxy.health_endpoints._health_endpoints import ( + _save_background_health_checks_to_db, + ) + + checked_by = ( + shared_health_manager.pod_id + if shared_health_manager is not None + else "background_health_check" + ) + start_time = time_module.time() + asyncio.create_task( + _save_background_health_checks_to_db( + prisma_client, + model_list, + healthy_endpoints, + unhealthy_endpoints, + start_time, + checked_by=checked_by, + ) + ) + + async def _run_background_health_check(): """ Periodically run health checks in the background on the endpoints. @@ -2053,25 +2112,6 @@ async def _run_background_health_check(): "cycle_id": cycle_id, } - async def _run_direct_health_check_with_instrumentation(): - try: - return await perform_health_check( - model_list=_llm_model_list, - details=health_check_details, - max_concurrency=health_check_concurrency, - instrumentation_context=instrumentation_context, - ) - except TypeError as e: - if "instrumentation_context" not in str(e): - raise - # Backward compatibility for monkeypatched or wrapped callables - # that do not accept instrumentation_context. - return await perform_health_check( - model_list=_llm_model_list, - details=health_check_details, - max_concurrency=health_check_concurrency, - ) - # Use shared health check if available, otherwise fall back to direct health check # Convert health_check_details to bool for perform_shared_health_check (defaults to True if None) details_bool = ( @@ -2094,11 +2134,21 @@ async def _run_background_health_check(): str(e), ) healthy_endpoints, unhealthy_endpoints = ( - await _run_direct_health_check_with_instrumentation() + await _run_direct_health_check_with_instrumentation( + _llm_model_list, + health_check_details, + health_check_concurrency, + instrumentation_context, + ) ) else: healthy_endpoints, unhealthy_endpoints = ( - await _run_direct_health_check_with_instrumentation() + await _run_direct_health_check_with_instrumentation( + _llm_model_list, + health_check_details, + health_check_concurrency, + instrumentation_context, + ) ) # Update the global variable with the health check results @@ -2127,32 +2177,13 @@ async def _run_background_health_check(): ) # Save background health checks to database (non-blocking) - if prisma_client is not None: - import time as time_module - - from litellm.proxy.health_endpoints._health_endpoints import ( - _save_background_health_checks_to_db, - ) - - # Use pod_id or a system identifier for checked_by if shared health check is enabled - checked_by = None - if shared_health_manager is not None: - checked_by = shared_health_manager.pod_id - else: - # Use a system identifier for background health checks - checked_by = "background_health_check" - - start_time = time_module.time() - asyncio.create_task( - _save_background_health_checks_to_db( - prisma_client, - _llm_model_list, - healthy_endpoints, - unhealthy_endpoints, - start_time, - checked_by=checked_by, - ) - ) + _schedule_background_health_check_db_save( + prisma_client, + shared_health_manager, + _llm_model_list, + healthy_endpoints, + unhealthy_endpoints, + ) await asyncio.sleep(health_check_interval) diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index f47fd6323f0..871b18062ff 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -365,6 +365,7 @@ def rerank( # noqa: PLR0915 max_chunks_per_doc=max_chunks_per_doc, _is_async=_is_async, optional_params=optional_params.model_dump(exclude_unset=True), + timeout=optional_params.timeout, api_base=api_base, extra_headers=merged_headers, logging_obj=litellm_logging_obj, diff --git a/tests/test_litellm/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py b/tests/test_litellm/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py index a8ac680908e..d6dc4bfa48d 100644 --- a/tests/test_litellm/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py +++ b/tests/test_litellm/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py @@ -237,6 +237,83 @@ async def test_bedrock_rerank_header_forwarding_async(model): pytest.fail(f"Failed to forward headers to {model}: {str(e)}") +def test_bedrock_rerank_timeout_sync(): + """ + Test that the timeout parameter is passed through to the HTTP client for Bedrock rerank (sync). + """ + client = HTTPHandler() + model = "bedrock/arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0" + mock_credentials_info = create_mock_credentials() + + with patch.object(client, "post") as mock_post, \ + patch("litellm.llms.bedrock.rerank.handler.BedrockRerankHandler._get_boto_credentials_from_optional_params", return_value=mock_credentials_info), \ + patch("botocore.auth.SigV4Auth") as mock_sigv4: + + mock_sigv4.return_value = MagicMock() + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(bedrock_rerank_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_response.raise_for_status = lambda: None + mock_post.return_value = mock_response + + litellm.rerank( + model=model, + query=test_query, + documents=test_documents, + top_n=3, + client=client, + timeout=0.001, + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint="https://bedrock-runtime.us-east-1.amazonaws.com", + ) + + assert mock_post.called + call_kwargs = mock_post.call_args.kwargs + assert call_kwargs.get("timeout") == 0.001, ( + f"Expected timeout=0.001, got timeout={call_kwargs.get('timeout')}" + ) + + +@pytest.mark.asyncio +async def test_bedrock_rerank_timeout_async(): + """ + Test that the timeout parameter is passed through to the HTTP client for Bedrock rerank (async). + """ + client = AsyncHTTPHandler() + model = "bedrock/arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0" + mock_credentials_info = create_mock_credentials() + + with patch.object(client, "post", new_callable=AsyncMock) as mock_post, \ + patch("litellm.llms.bedrock.rerank.handler.BedrockRerankHandler._get_boto_credentials_from_optional_params", return_value=mock_credentials_info), \ + patch("botocore.auth.SigV4Auth") as mock_sigv4: + + mock_sigv4.return_value = MagicMock() + mock_response = AsyncMock() + mock_response.status_code = 200 + mock_response.text = json.dumps(bedrock_rerank_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_response.raise_for_status = lambda: None + mock_post.return_value = mock_response + + await litellm.arerank( + model=model, + query=test_query, + documents=test_documents, + top_n=3, + client=client, + timeout=0.001, + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint="https://bedrock-runtime.us-east-1.amazonaws.com", + ) + + assert mock_post.called + call_kwargs = mock_post.call_args.kwargs + assert call_kwargs.get("timeout") == 0.001, ( + f"Expected timeout=0.001, got timeout={call_kwargs.get('timeout')}" + ) + + def test_bedrock_rerank_extra_headers_and_headers_merge(): """ Test that both extra_headers and headers parameters are correctly merged for Bedrock rerank. From 1c48d8fda7512b386558044cec5bfbe18e22ee5c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 24 Feb 2026 23:37:09 +0530 Subject: [PATCH 07/12] Add gpt-5.3-codex in model cost map --- ...odel_prices_and_context_window_backup.json | 33 +++++++++++++++++++ model_prices_and_context_window.json | 33 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3909ce4c8b0..00b960a9f7a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -20562,6 +20562,39 @@ "supports_tool_choice": true, "supports_vision": true }, + "gpt-5.3-codex": { + "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_priority": 3.5e-07, + "input_cost_per_token": 1.75e-06, + "input_cost_per_token_priority": 3.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1.4e-05, + "output_cost_per_token_priority": 2.8e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, "gpt-5-mini": { "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3909ce4c8b0..00b960a9f7a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -20562,6 +20562,39 @@ "supports_tool_choice": true, "supports_vision": true }, + "gpt-5.3-codex": { + "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_priority": 3.5e-07, + "input_cost_per_token": 1.75e-06, + "input_cost_per_token_priority": 3.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1.4e-05, + "output_cost_per_token_priority": 2.8e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, "gpt-5-mini": { "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, From aded14a55ac973c91559e49fcef4d267cbc17229 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 25 Feb 2026 01:04:12 +0530 Subject: [PATCH 08/12] Fix release version for gpt-5.3-codex --- docs/my-website/blog/gpt_5_3_codex/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/my-website/blog/gpt_5_3_codex/index.md b/docs/my-website/blog/gpt_5_3_codex/index.md index 321e573cccf..5c43d287bce 100644 --- a/docs/my-website/blog/gpt_5_3_codex/index.md +++ b/docs/my-website/blog/gpt_5_3_codex/index.md @@ -44,7 +44,7 @@ Important: ## Docker Image ```bash -docker pull ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 +docker pull ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3 ``` ## Usage @@ -68,7 +68,7 @@ docker run -d \ -p 4000:4000 \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -v $(pwd)/config.yaml:/app/config.yaml \ - ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \ + ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3 \ --config /app/config.yaml ``` From 74abf0c8e6b6144032dc386972626b2b6ee22a98 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 25 Feb 2026 01:19:10 +0530 Subject: [PATCH 09/12] Fix phase docs link --- docs/my-website/blog/gpt_5_3_codex/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/blog/gpt_5_3_codex/index.md b/docs/my-website/blog/gpt_5_3_codex/index.md index 5c43d287bce..5b7773cf8aa 100644 --- a/docs/my-website/blog/gpt_5_3_codex/index.md +++ b/docs/my-website/blog/gpt_5_3_codex/index.md @@ -29,7 +29,7 @@ LiteLLM now supports GPT-5.3-Codex on Day 0, including support for the new assis `phase` appears on assistant output items and helps distinguish preamble/commentary turns from final closeout responses. -Reference: [Phase parameter docs](https://developers-site-git-alphas-venusaur-api-openai.vercel.app/alphas/venusaur-api/phase-parameter) +Reference: [Phase parameter docs](https://developers.openai.com/api/reference/overview) Supported values: - `null` From 5d291c739fbc493e3d2dae593add5ba6eb221c0a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 25 Feb 2026 01:21:38 +0530 Subject: [PATCH 10/12] Fix phase docs link --- docs/my-website/blog/gpt_5_3_codex/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/blog/gpt_5_3_codex/index.md b/docs/my-website/blog/gpt_5_3_codex/index.md index 5b7773cf8aa..850586538f6 100644 --- a/docs/my-website/blog/gpt_5_3_codex/index.md +++ b/docs/my-website/blog/gpt_5_3_codex/index.md @@ -66,7 +66,7 @@ model_list: ```bash docker run -d \ -p 4000:4000 \ - -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -e ANTHROPIC_API_KEY=$OPENAI_API_KEY \ -v $(pwd)/config.yaml:/app/config.yaml \ ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3 \ --config /app/config.yaml From e44b9b6b3584710a948365268d64676e2177adab Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 24 Feb 2026 11:51:42 -0800 Subject: [PATCH 11/12] feat(prometheus): add opt-in stream label to litellm_proxy_total_requests_metric (#22023) Set prometheus_emit_stream_label: true in litellm_settings to emit a stream label (True/False/None) on litellm_proxy_total_requests_metric. Opt-in to avoid breaking cardinality on existing deployments. --- docs/my-website/docs/proxy/prometheus.md | 26 +++++- litellm/__init__.py | 1 + litellm/integrations/prometheus.py | 6 ++ litellm/types/integrations/prometheus.py | 25 ++++-- .../test_prometheus_stream_label.py | 81 +++++++++++++++++++ 5 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 tests/test_litellm/integrations/test_prometheus_stream_label.py diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index 93a0675f097..18a139d1d29 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -122,7 +122,7 @@ Use this to track overall LiteLLM Proxy usage. | Metric Name | Description | |----------------------|--------------------------------------| | `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "user_email", "exception_status", "exception_class", "route", "model_id"` | -| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route", "model_id"` | +| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route", "model_id"`. Optionally includes `"stream"` — see [Emit Stream Label](#emit-stream-label). | ### Callback Logging Metrics @@ -214,9 +214,31 @@ litellm_settings: ``` +### Emit Stream Label + +Add a `stream` label to `litellm_proxy_total_requests_metric` to split requests by streaming vs. non-streaming. Disabled by default. + +```yaml title="config.yaml" +litellm_settings: + callbacks: ["prometheus"] + prometheus_emit_stream_label: true +``` + +When enabled, `litellm_proxy_total_requests_metric` gains a `stream` label with values `"True"`, `"False"`, or `"None"`. + +``` +litellm_proxy_total_requests_metric{..., stream="True"} 42 +litellm_proxy_total_requests_metric{..., stream="False"} 100 +``` + +:::note +This label is opt-in because adding a new label to an existing metric changes its cardinality and breaks existing Prometheus queries / Grafana dashboards that target this metric. Enable it only on fresh deployments or when you are ready to update your dashboards. +::: + + ## [BETA] Custom Metrics -Track custom metrics on prometheus on all events mentioned above. +Track custom metrics on prometheus on all events mentioned above. ### Custom Metadata Labels diff --git a/litellm/__init__.py b/litellm/__init__.py index 1e74b5692e4..6e42f2c1ea5 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -374,6 +374,7 @@ enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None custom_prometheus_metadata_labels: List[str] = [] custom_prometheus_tags: List[str] = [] prometheus_metrics_config: Optional[List] = None +prometheus_emit_stream_label: bool = False disable_add_prefix_to_prompt: bool = ( False # used by anthropic, to disable adding prefix to prompt ) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 4c7afd5a57c..08db77e8571 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -974,6 +974,9 @@ class PrometheusLogger(CustomLogger): ), client_ip=standard_logging_payload["metadata"].get("requester_ip_address"), user_agent=standard_logging_payload["metadata"].get("user_agent"), + stream=str(standard_logging_payload.get("stream")) + if litellm.prometheus_emit_stream_label + else None, ) if ( @@ -1624,6 +1627,9 @@ class PrometheusLogger(CustomLogger): client_ip=_metadata.get("requester_ip_address"), user_agent=_metadata.get("user_agent"), model_id=model_id, + stream=str(request_data.get("stream")) + if litellm.prometheus_emit_stream_label + else None, ) _labels = prometheus_label_factory( supported_enum_labels=self.get_labels_for_metric( diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index fd788af9ac1..482b87085dd 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -55,22 +55,21 @@ def _sanitize_prometheus_label_value(value: Optional[Any]) -> Optional[str]: return None # Coerce non-string values (int, bool, etc.) to str before sanitizing - if not isinstance(value, str): - value = str(value) + str_value: str = value if isinstance(value, str) else str(value) # Remove Unicode line/paragraph separators that break text format - value = value.replace("\u2028", "").replace("\u2029", "") + str_value = str_value.replace("\u2028", "").replace("\u2029", "") # Remove carriage returns - value = value.replace("\r", "") + str_value = str_value.replace("\r", "") # Replace newlines with spaces - value = value.replace("\n", " ") + str_value = str_value.replace("\n", " ") # Escape backslashes and double quotes per Prometheus exposition format - value = value.replace("\\", "\\\\").replace('"', '\\"') + str_value = str_value.replace("\\", "\\\\").replace('"', '\\"') - return value + return str_value @dataclass @@ -185,6 +184,7 @@ class UserAPIKeyLabelNames(Enum): CLIENT_IP = "client_ip" USER_AGENT = "user_agent" CALLBACK_NAME = "callback_name" + STREAM = "stream" DEFINED_PROMETHEUS_METRICS = Literal[ @@ -638,6 +638,14 @@ class PrometheusMetricLabels: ] ) + # Conditionally add stream label to litellm_proxy_total_requests_metric + if ( + label_name == "litellm_proxy_total_requests_metric" + and litellm.prometheus_emit_stream_label is True + and UserAPIKeyLabelNames.STREAM.value not in default_labels + ): + custom_labels.append(UserAPIKeyLabelNames.STREAM.value) + return default_labels + custom_labels @@ -709,6 +717,9 @@ class UserAPIKeyLabelValues(BaseModel): user_agent: Annotated[ Optional[str], Field(..., alias=UserAPIKeyLabelNames.USER_AGENT.value) ] = None + stream: Annotated[ + Optional[str], Field(..., alias=UserAPIKeyLabelNames.STREAM.value) + ] = None class PrometheusMetricsConfig(BaseModel): diff --git a/tests/test_litellm/integrations/test_prometheus_stream_label.py b/tests/test_litellm/integrations/test_prometheus_stream_label.py new file mode 100644 index 00000000000..a00a468e0fb --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_stream_label.py @@ -0,0 +1,81 @@ +""" +Unit tests for prometheus_emit_stream_label opt-in setting. + +Tests that: +- stream label is NOT added to litellm_proxy_total_requests_metric by default +- stream label IS added when litellm.prometheus_emit_stream_label = True +- stream value is populated correctly from standard_logging_payload +""" +import pytest + +import litellm +from litellm.types.integrations.prometheus import ( + PrometheusMetricLabels, + UserAPIKeyLabelNames, +) + + +def test_stream_label_not_present_by_default(): + """stream label should NOT appear in litellm_proxy_total_requests_metric unless opted in""" + litellm.prometheus_emit_stream_label = False + labels = PrometheusMetricLabels.get_labels("litellm_proxy_total_requests_metric") + assert UserAPIKeyLabelNames.STREAM.value not in labels + + +def test_stream_label_present_when_opted_in(): + """stream label SHOULD appear in litellm_proxy_total_requests_metric when opted in""" + litellm.prometheus_emit_stream_label = True + try: + labels = PrometheusMetricLabels.get_labels("litellm_proxy_total_requests_metric") + assert UserAPIKeyLabelNames.STREAM.value in labels + finally: + litellm.prometheus_emit_stream_label = False + + +def test_stream_label_not_in_other_metrics_when_opted_in(): + """stream label should NOT be added to other metrics even when opted in""" + litellm.prometheus_emit_stream_label = True + try: + other_metrics = [ + "litellm_proxy_failed_requests_metric", + "litellm_spend_metric", + "litellm_input_tokens_metric", + "litellm_output_tokens_metric", + "litellm_llm_api_latency_metric", + ] + for metric in other_metrics: + labels = PrometheusMetricLabels.get_labels(metric) + assert UserAPIKeyLabelNames.STREAM.value not in labels, ( + f"stream label should not be in {metric}" + ) + finally: + litellm.prometheus_emit_stream_label = False + + +def test_stream_label_name(): + """STREAM label name should be 'stream'""" + assert UserAPIKeyLabelNames.STREAM.value == "stream" + + +def test_user_api_key_label_values_has_stream_field(): + """UserAPIKeyLabelValues should accept stream field""" + from litellm.types.integrations.prometheus import UserAPIKeyLabelValues + + values = UserAPIKeyLabelValues(stream="True") + assert values.stream == "True" + + values_false = UserAPIKeyLabelValues(stream="False") + assert values_false.stream == "False" + + values_none = UserAPIKeyLabelValues() + assert values_none.stream is None + + +def test_stream_label_in_model_dump(): + """stream field appears in model_dump() output for use in prometheus_label_factory""" + from litellm.types.integrations.prometheus import UserAPIKeyLabelValues + + values = UserAPIKeyLabelValues(stream="True") + dumped = values.model_dump() + assert "stream" in dumped + assert dumped["stream"] == "True" From c343bfffdaa3e34e9324733d897a8e4db79ac9ec Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 24 Feb 2026 11:56:16 -0800 Subject: [PATCH 12/12] fix(router): emit x-litellm-overhead-duration-ms header for streaming requests (#22027) * fix(router): preserve _hidden_params in FallbackStreamWrapper so x-litellm-overhead-duration-ms is emitted for streaming requests * test(router): add regression test for FallbackStreamWrapper _hidden_params preservation --- litellm/router.py | 26 ++- .../proxy/test_common_request_processing.py | 201 ++++++++++++++++++ tests/test_litellm/test_router.py | 57 ++++- 3 files changed, 274 insertions(+), 10 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index ac2862da689..46d35352c37 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -163,7 +163,11 @@ from litellm.types.utils import ( ) from litellm.types.utils import ModelInfo from litellm.types.utils import ModelInfo as ModelMapInfo -from litellm.types.utils import ModelResponseStream, StandardLoggingPayload, Usage +from litellm.types.utils import ( + ModelResponseStream, + StandardLoggingPayload, + Usage, +) from litellm.utils import ( CustomStreamWrapper, EmbeddingResponse, @@ -1555,6 +1559,9 @@ class Router: logging_obj=model_response.logging_obj, ) self._async_generator = async_generator + # Preserve hidden params (including litellm_overhead_time_ms) from original response + if hasattr(model_response, "_hidden_params"): + self._hidden_params = model_response._hidden_params.copy() def __aiter__(self): return self @@ -6978,9 +6985,9 @@ class Router: raise ValueError("Deployment not found") ## GET BASE MODEL - base_model = deployment.get("model_info", {}).get("base_model", None) + base_model = (deployment.get("model_info") or {}).get("base_model", None) if base_model is None: - base_model = deployment.get("litellm_params", {}).get("base_model", None) + base_model = (deployment.get("litellm_params") or {}).get("base_model", None) model = base_model @@ -6995,7 +7002,7 @@ class Router: raise ValueError( f"Deployment missing valid litellm_params. " f"Got: {type(litellm_params_data).__name__}, " - f"deployment_id: {deployment.get('model_info', {}).get('id', 'unknown')}" + f"deployment_id: {(deployment.get('model_info') or {}).get('id', 'unknown')}" ) _model, custom_llm_provider, _, _ = litellm.get_llm_provider( model=litellm_params.model, @@ -7015,10 +7022,10 @@ class Router: if potential_models is not None: for potential_model in potential_models: try: - if potential_model.get("model_info", {}).get( + if (potential_model.get("model_info") or {}).get( "id" - ) == deployment.get("model_info", {}).get("id"): - model = potential_model.get("litellm_params", {}).get( + ) == (deployment.get("model_info") or {}).get("id"): + model = (potential_model.get("litellm_params") or {}).get( "model" ) break @@ -7039,9 +7046,10 @@ class Router: model_info = litellm.get_model_info(model=model_info_name) ## CHECK USER SET MODEL INFO - user_model_info = deployment.get("model_info", {}) + user_model_info = deployment.get("model_info") or {} - model_info.update(user_model_info) + if model_info is not None: + model_info.update(user_model_info) return model_info diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 977304f732b..bf794478f10 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -1,4 +1,6 @@ import copy +import datetime +from typing import AsyncGenerator from unittest.mock import AsyncMock, MagicMock import pytest @@ -1348,3 +1350,202 @@ class TestOverrideOpenAIResponseModel: # Verify the model was not changed assert response_obj.model == fallback_model + + +class TestStreamingOverheadHeader: + """ + Tests that x-litellm-overhead-duration-ms is emitted in streaming responses. + + Regression tests for: streaming requests not including overhead header. + """ + + def test_get_custom_headers_includes_overhead_when_set(self): + """ + get_custom_headers() returns x-litellm-overhead-duration-ms + when litellm_overhead_time_ms is in hidden_params. + """ + mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) + mock_user_api_key_dict.tpm_limit = None + mock_user_api_key_dict.rpm_limit = None + mock_user_api_key_dict.max_budget = None + mock_user_api_key_dict.spend = 0.0 + mock_user_api_key_dict.allowed_model_region = None + + hidden_params = { + "litellm_overhead_time_ms": 42.5, + "_response_ms": 500.0, + "model_id": "test-model-id", + "api_base": "https://api.openai.com", + } + + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=mock_user_api_key_dict, + call_id="test-call-id", + model_id="test-model-id", + cache_key="", + api_base="https://api.openai.com", + version="1.0.0", + response_cost=0.001, + model_region="", + hidden_params=hidden_params, + ) + + assert "x-litellm-overhead-duration-ms" in headers + assert headers["x-litellm-overhead-duration-ms"] == "42.5" + + def test_get_custom_headers_omits_overhead_when_none(self): + """ + get_custom_headers() omits x-litellm-overhead-duration-ms + when litellm_overhead_time_ms is not in hidden_params. + """ + mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) + mock_user_api_key_dict.tpm_limit = None + mock_user_api_key_dict.rpm_limit = None + mock_user_api_key_dict.max_budget = None + mock_user_api_key_dict.spend = 0.0 + mock_user_api_key_dict.allowed_model_region = None + + hidden_params = { + "_response_ms": 500.0, + "model_id": "test-model-id", + } + + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=mock_user_api_key_dict, + call_id="test-call-id", + model_id="test-model-id", + cache_key="", + api_base="https://api.openai.com", + version="1.0.0", + response_cost=0.001, + model_region="", + hidden_params=hidden_params, + ) + + # Should be absent (None gets filtered by exclude_values) + assert "x-litellm-overhead-duration-ms" not in headers + + def test_update_response_metadata_sets_overhead_on_stream_wrapper(self): + """ + update_response_metadata() sets litellm_overhead_time_ms on + a streaming response's _hidden_params when llm_api_duration_ms is available. + """ + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + update_response_metadata, + ) + + # Mock the logging object with llm_api_duration_ms set + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = { + "llm_api_duration_ms": 200.0, + "litellm_params": {}, + } + mock_logging_obj.caching_details = None + mock_logging_obj.callback_duration_ms = None + mock_logging_obj.litellm_call_id = "test-call-id" + mock_logging_obj._response_cost_calculator = MagicMock(return_value=0.001) + + # Simulate a streaming result object with _hidden_params (like CustomStreamWrapper) + stream_result = MagicMock() + stream_result._hidden_params = { + "model_id": "test-model-id", + "api_base": "https://api.openai.com", + "additional_headers": {}, + } + + start_time = datetime.datetime.now() - datetime.timedelta(milliseconds=300) + end_time = datetime.datetime.now() + + update_response_metadata( + result=stream_result, + logging_obj=mock_logging_obj, + model="gpt-4o", + kwargs={}, + start_time=start_time, + end_time=end_time, + ) + + assert "litellm_overhead_time_ms" in stream_result._hidden_params + overhead = stream_result._hidden_params["litellm_overhead_time_ms"] + assert overhead is not None + assert isinstance(overhead, float) + # overhead = total_response_ms (~300ms) - llm_api_duration_ms (200ms) = ~100ms + assert overhead > 0 + + @pytest.mark.asyncio + async def test_streaming_response_includes_overhead_header(self): + """ + StreamingResponse returned by create_response() includes + x-litellm-overhead-duration-ms in its headers. + """ + + async def mock_generator() -> AsyncGenerator[str, None]: + yield 'data: {"id":"chatcmpl-test","choices":[{"delta":{"content":"hi"}}]}\n\n' + yield "data: [DONE]\n\n" + + headers = { + "x-litellm-overhead-duration-ms": "42.5", + "x-litellm-call-id": "test-call-id", + "x-litellm-model-id": "test-model-id", + } + + response = await create_response( + generator=mock_generator(), + media_type="text/event-stream", + headers=headers, + ) + + assert isinstance(response, StreamingResponse) + assert response.headers.get("x-litellm-overhead-duration-ms") == "42.5" + + def test_streaming_overhead_header_in_custom_headers_from_stream_hidden_params( + self, + ): + """ + Verifies that when get_custom_headers() is called with a streaming + response's hidden_params (containing litellm_overhead_time_ms), + the x-litellm-overhead-duration-ms header is correctly populated. + + This tests the critical path: update_response_metadata sets the value + → get_custom_headers reads it → StreamingResponse header is set. + """ + mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) + mock_user_api_key_dict.tpm_limit = None + mock_user_api_key_dict.rpm_limit = None + mock_user_api_key_dict.max_budget = None + mock_user_api_key_dict.spend = 0.0 + mock_user_api_key_dict.allowed_model_region = None + + # This is what CustomStreamWrapper._hidden_params looks like after + # update_response_metadata() has been called on it + hidden_params = { + "model_id": "openai-gpt4o-deployment", + "api_base": "https://api.openai.com", + "additional_headers": {}, + "litellm_overhead_time_ms": 55.3, # set by update_response_metadata + "_response_ms": 280.0, + "litellm_call_id": "test-call-id", + "response_cost": 0.002, + "cache_key": None, + "fastest_response_batch_completion": None, + "callback_duration_ms": None, + } + + custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=mock_user_api_key_dict, + call_id="test-call-id", + model_id=hidden_params.get("model_id"), + cache_key=hidden_params.get("cache_key") or "", + api_base=hidden_params.get("api_base") or "", + version="1.0.0", + response_cost=hidden_params.get("response_cost"), + model_region="", + hidden_params=hidden_params, + ) + + # The overhead header must be present and correct + assert "x-litellm-overhead-duration-ms" in custom_headers, ( + "x-litellm-overhead-duration-ms header must be emitted during streaming. " + "It was missing — this is the streaming overhead header regression." + ) + assert custom_headers["x-litellm-overhead-duration-ms"] == "55.3" diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 5542cdf8be4..5732deda6fb 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1297,6 +1297,61 @@ async def test_acompletion_streaming_iterator_edge_cases(): print("✓ Edge case tests passed!") +@pytest.mark.asyncio +async def test_acompletion_streaming_iterator_preserves_hidden_params(): + """ + Regression test: FallbackStreamWrapper must copy _hidden_params from the + original CustomStreamWrapper so that x-litellm-overhead-duration-ms (and + other hidden params) are present in the proxy response headers for streaming. + """ + from unittest.mock import MagicMock + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4", "api_key": "fake-key"}, + } + ], + ) + + # Simulate a CustomStreamWrapper that already has timing metadata set by + # update_response_metadata (litellm_overhead_time_ms, _response_ms, etc.) + mock_response = MagicMock() + mock_response.model = "gpt-4" + mock_response.custom_llm_provider = "openai" + mock_response.logging_obj = MagicMock() + mock_response._hidden_params = { + "litellm_overhead_time_ms": 12.34, + "_response_ms": 500.0, + "litellm_call_id": "test-call-id", + "api_base": "https://api.openai.com", + "additional_headers": {}, + } + + # Make the mock iterable (yields nothing — we only care about hidden_params) + async def _empty(): + return + yield # make it an async generator + + mock_response.__aiter__ = lambda self: _empty().__aiter__() + + result = await router._acompletion_streaming_iterator( + model_response=mock_response, + messages=[{"role": "user", "content": "hi"}], + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + + # The returned FallbackStreamWrapper must carry the original _hidden_params + assert hasattr(result, "_hidden_params"), "result must have _hidden_params" + assert result._hidden_params.get("litellm_overhead_time_ms") == 12.34, ( + "litellm_overhead_time_ms must be preserved — " + "this is what drives x-litellm-overhead-duration-ms in streaming responses" + ) + assert result._hidden_params.get("litellm_call_id") == "test-call-id" + assert result._hidden_params.get("_response_ms") == 500.0 + + @pytest.mark.asyncio async def test_async_function_with_fallbacks_common_utils(): """Test the async_function_with_fallbacks_common_utils method""" @@ -1858,7 +1913,7 @@ def test_get_deployment_credentials_with_provider_resolves_credential_name(): litellm_credential_name to actual credential values (for UI-created models). """ from litellm.types.utils import CredentialItem - + # Setup credential list with a test credential litellm.credential_list = [ CredentialItem(