From f096e4c10bfc1f8db91f3ea3dc649aac1ce98a2b Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 02:27:07 +0000 Subject: [PATCH 01/23] docs: require a user flow and live-proxy proof in bug reports Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/bug_report.yml | 60 ++++++++++++++++++++++----- CLAUDE.md | 2 + 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 665f8456f0b..4c1cc637f08 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,26 +27,64 @@ body: validations: required: true - type: textarea - id: steps-to-reproduce + id: user-flow attributes: - label: Steps to Reproduce - description: Please provide a numbered list of the exact steps to reproduce this bug (include a curl/python snippet to reproduce it). Number each step (1., 2., 3., ...) in the order you performed them. - placeholder: | - 1. config.yaml file/ .env file/ etc. - 2. Run the following code... - 3. Observe the error... + label: User Flow + description: Two numbered lists walking the same end user through the same task, one before a hypothetical fix and one after. Keep the guidance comments in the box while you fill it in, they explain every rule. value: | + + + Before a (hypothetical) fix: + + 1. + 2. + 3. + + After a (hypothetical) fix: + 1. 2. 3. validations: required: true - type: textarea - id: logs + id: proof-of-bug attributes: - label: Relevant log output - description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. - render: shell + label: Proof the bug occurs + description: Paste the commands you ran and their output, captured against a live proxy with no mocks. Keep the guidance comments in the box while you fill it in, they explain every rule. + value: | + + + validations: + required: true - type: dropdown id: component attributes: diff --git a/CLAUDE.md b/CLAUDE.md index 436fa33fa41..e4f027715cc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,6 +31,8 @@ When creating PRs, don't set base to `main`. `litellm_internal_staging` is the d When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule +The same goes for filing a bug report: treat the comments and imperative instructions inside @.github/ISSUE_TEMPLATE/bug_report.yml as rules to follow, not just layout, and read that file from disk before writing an issue body so no stripped HTML comment escapes you + If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR From 7e8faf926740e48bf12f10ffb223534d2c3df233 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 02:33:10 +0000 Subject: [PATCH 02/23] docs: ask bug reports for the config and version behind the proof Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 4c1cc637f08..7b3c9b072d8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -78,7 +78,7 @@ body: From 7653532e89e134206002672ebb545ba6afc6db26 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 02:35:44 +0000 Subject: [PATCH 03/23] docs: spell out secret redaction in the bug report proof Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/bug_report.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 7b3c9b072d8..cedf04a40d6 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -78,10 +78,9 @@ body: + For UI bugs: include screenshots and the page URLs you were on. Scrub keys and tokens out of screenshots too, they show up in headers, request panels, and the Admin UI --> validations: required: true From 550682d5f8e7a41bc09e54a96179edb8eeada9d7 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 02:36:11 +0000 Subject: [PATCH 04/23] docs: redact only sensitive env vars in bug report proof Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index cedf04a40d6..0044c2f2b88 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -78,7 +78,7 @@ body: From f8a5d6a6a1109fc9903960c2c5f3623f99fd820a Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 02:49:57 +0000 Subject: [PATCH 05/23] docs: make an unfilled bug report proof visibly empty and attested Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/bug_report.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 0044c2f2b88..86f66764ab0 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -82,8 +82,21 @@ body: If the bug applies to more than one of the LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), include proof for every one of them, not just one For UI bugs: include screenshots and the page URLs you were on. Scrub keys and tokens out of screenshots too, they show up in headers, request panels, and the Admin UI --> + Config / setup the proxy ran with: + + Version or commit: + + Commands and their full output: + validations: required: true + - type: checkboxes + id: proof-attestation + attributes: + label: About that proof + options: + - label: It came from a live proxy I ran myself, with no mocks, and shows the real commands and their output rather than a `pytest` run + required: true - type: dropdown id: component attributes: From 7a17735473632d6002cdf38f27824f05e12db97b Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 02:56:13 +0000 Subject: [PATCH 06/23] fix(triage): treat an unfilled bug report scaffold as missing proof Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/scripts/triage_with_llm.py | 4 +++- tests/test_litellm/test_github_triage_with_llm.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/scripts/triage_with_llm.py b/.github/scripts/triage_with_llm.py index d2536058e01..a7bd145dbc8 100644 --- a/.github/scripts/triage_with_llm.py +++ b/.github/scripts/triage_with_llm.py @@ -582,7 +582,9 @@ def build_issue_prompt(*, title: str, body: str) -> str: Commands whose external dependencies (LLM provider, DB, network) are mocked or stubbed do NOT count. Prose-only "steps to reproduce" with no run output, video, or - screenshot do NOT satisfy (1). + screenshot do NOT satisfy (1). An unfilled template scaffold + (bare headings such as "Version or commit:" with nothing under + them, empty numbered lists) counts as absent, not as evidence. (2) Expected vs. actual behavior (`has_expected_vs_actual`). FAIL the bug report if either (1) or (2) is missing. Do not bias diff --git a/tests/test_litellm/test_github_triage_with_llm.py b/tests/test_litellm/test_github_triage_with_llm.py index f50cf126c36..300fd7c0710 100644 --- a/tests/test_litellm/test_github_triage_with_llm.py +++ b/tests/test_litellm/test_github_triage_with_llm.py @@ -672,6 +672,11 @@ class TestBuildPrompts: assert "mocked or stubbed" in normalized # Prose-only steps are explicitly insufficient now. assert "steps to reproduce" in normalized + # An unedited issue-form scaffold must not read as evidence: the proof + # field ships with visible headings, so the judge has to be told that + # bare headings with nothing under them count as absent. + assert "unfilled template scaffold" in normalized + assert "counts as absent, not as evidence" in normalized def test_should_not_crash_when_pr_body_contains_curly_braces(self, triage_module): """User-supplied content with `{` / `}` must NOT be re-parsed by From 5643a59aa4e0a7322a14ac0645dad21631cd66f8 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 02:58:21 +0000 Subject: [PATCH 07/23] fix: move bug report guidance out of prefilled values so required means filled Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/bug_report.yml | 61 ++++++++++----------------- CLAUDE.md | 2 +- 2 files changed, 23 insertions(+), 40 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 86f66764ab0..7e08f9297f8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -23,71 +23,54 @@ body: label: What happened? description: Also tell us, what did you expect to happen? placeholder: Tell us what you see! - value: "A bug happened!" validations: required: true - type: textarea id: user-flow attributes: label: User Flow - description: Two numbered lists walking the same end user through the same task, one before a hypothetical fix and one after. Keep the guidance comments in the box while you fill it in, they explain every rule. - value: | - - - Before a (hypothetical) fix: - - 1. - 2. - 3. - - After a (hypothetical) fix: - - 1. - 2. - 3. validations: required: true - type: textarea id: proof-of-bug attributes: label: Proof the bug occurs - description: Paste the commands you ran and their output, captured against a live proxy with no mocks. Keep the guidance comments in the box while you fill it in, they explain every rule. - value: | - + description: | + The commands (e.g., curl) and their full output, screenshots, or a screen recording demonstrating that the bug happens. Every rule below applies. + - The proof must be completely e2e with no mocks, against a live proxy you ran yourself (e.g., `litellm --config config.yaml --detailed_debug` on localhost:4000), hitting real LLM provider APIs and costing real $ where the bug involves a provider call. `pytest` commands are not enough + - Show exactly what the end user sees or does, matching the User Flow above step for step + - Start with the config.yaml (or SDK setup) and any env vars the proxy ran with, then the exact version or commit hash the proof was captured at, so a maintainer can stand up the same proxy before running your commands. Keep the real values for env vars that aren't sensitive, they are often the reason the bug happens, and redact only the secrets: never paste a real API key, virtual key, database URL, or other credential, here or anywhere else in the issue + - If the bug applies to more than one of the LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), include proof for every one of them, not just one + - For UI bugs: include screenshots and the page URLs you were on. Scrub keys and tokens out of screenshots too, they show up in headers, request panels, and the Admin UI + placeholder: | Config / setup the proxy ran with: Version or commit: Commands and their full output: - validations: required: true - type: checkboxes diff --git a/CLAUDE.md b/CLAUDE.md index e4f027715cc..3941060afa2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,7 +31,7 @@ When creating PRs, don't set base to `main`. `litellm_internal_staging` is the d When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule -The same goes for filing a bug report: treat the comments and imperative instructions inside @.github/ISSUE_TEMPLATE/bug_report.yml as rules to follow, not just layout, and read that file from disk before writing an issue body so no stripped HTML comment escapes you +The same goes for filing a bug report: treat every field's `description` and `placeholder` in @.github/ISSUE_TEMPLATE/bug_report.yml as rules to follow, not just layout, and read that file from disk before writing an issue body, since the rendered form and any copy injected into your context can drop or reflow that guidance If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank From 0a41b19e5b1d669a0ec59206b8235b128d4e6889 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:33:22 -0700 Subject: [PATCH 08/23] chore: make it more brief --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3941060afa2..0e63b86036d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,7 +31,7 @@ When creating PRs, don't set base to `main`. `litellm_internal_staging` is the d When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule -The same goes for filing a bug report: treat every field's `description` and `placeholder` in @.github/ISSUE_TEMPLATE/bug_report.yml as rules to follow, not just layout, and read that file from disk before writing an issue body, since the rendered form and any copy injected into your context can drop or reflow that guidance +Same applies for filing bug reports and .github/ISSUE_TEMPLATE/bug_report.yml If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank From 10d4213453bcc9bd321892df9682a1f753d70e77 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 16:25:52 +0000 Subject: [PATCH 09/23] chore: drop the bug report proof attestation and tighten its wording Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/bug_report.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 7e08f9297f8..48818144caa 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -37,7 +37,7 @@ body: - Every step is something the user does or observes: the HTTP method and full URL they hit, what they sent, and what visibly came back (status code, error text, the shape of an ID). UI steps name the page URL and what is on screen - No LiteLLM internals: never name functions, files, DB tables, config classes, hooks, callbacks, or code paths. "The upload hands back an ID that looks like OpenAI's own `file-abc123` instead of the scrambled one the gateway returned" is right, "no managed-file row was registered" is wrong - Keep the two lists step-for-step identical until they diverge, so the broken step is obvious - - If the bug has a security or authorization consequence, end each list with what another user can or could no longer do + - If the bug has a security or authorization consequence, end each list with what another user can do that they shouldn't be able to, and what they could no longer do after a fix placeholder: | Before a (hypothetical) fix: a developer whose app streams chat completions gets no token counts back, so their cost dashboard reads zero @@ -60,11 +60,11 @@ body: description: | The commands (e.g., curl) and their full output, screenshots, or a screen recording demonstrating that the bug happens. Every rule below applies. - - The proof must be completely e2e with no mocks, against a live proxy you ran yourself (e.g., `litellm --config config.yaml --detailed_debug` on localhost:4000), hitting real LLM provider APIs and costing real $ where the bug involves a provider call. `pytest` commands are not enough + - The proof must be completely e2e with no mocks, against a live proxy you ran yourself (e.g., `litellm --config config.yaml --detailed_debug` on localhost:4000), hitting real LLM provider APIs, costing real $$$ if needed, where the bug involves a provider call. `pytest` commands are not enough - Show exactly what the end user sees or does, matching the User Flow above step for step - Start with the config.yaml (or SDK setup) and any env vars the proxy ran with, then the exact version or commit hash the proof was captured at, so a maintainer can stand up the same proxy before running your commands. Keep the real values for env vars that aren't sensitive, they are often the reason the bug happens, and redact only the secrets: never paste a real API key, virtual key, database URL, or other credential, here or anywhere else in the issue - If the bug applies to more than one of the LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), include proof for every one of them, not just one - - For UI bugs: include screenshots and the page URLs you were on. Scrub keys and tokens out of screenshots too, they show up in headers, request panels, and the Admin UI + - For UI bugs: include screenshots and the page URLs you were on. Scrub keys and tokens out of screenshots too (for example, the virtual key is briefly shown in the panel right after you create a virtual key) placeholder: | Config / setup the proxy ran with: @@ -73,13 +73,6 @@ body: Commands and their full output: validations: required: true - - type: checkboxes - id: proof-attestation - attributes: - label: About that proof - options: - - label: It came from a live proxy I ran myself, with no mocks, and shows the real commands and their output rather than a `pytest` run - required: true - type: dropdown id: component attributes: From ad1ff191952c91c6c03cd0837637f12cc82ab4a6 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 16:31:19 +0000 Subject: [PATCH 10/23] chore: say real $ instead of $$$ in the bug report proof rules Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 48818144caa..b93e4add9a7 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -60,7 +60,7 @@ body: description: | The commands (e.g., curl) and their full output, screenshots, or a screen recording demonstrating that the bug happens. Every rule below applies. - - The proof must be completely e2e with no mocks, against a live proxy you ran yourself (e.g., `litellm --config config.yaml --detailed_debug` on localhost:4000), hitting real LLM provider APIs, costing real $$$ if needed, where the bug involves a provider call. `pytest` commands are not enough + - The proof must be completely e2e with no mocks, against a live proxy you ran yourself (e.g., `litellm --config config.yaml --detailed_debug` on localhost:4000), hitting real LLM provider APIs, costing real $ if needed, where the bug involves a provider call. `pytest` commands are not enough - Show exactly what the end user sees or does, matching the User Flow above step for step - Start with the config.yaml (or SDK setup) and any env vars the proxy ran with, then the exact version or commit hash the proof was captured at, so a maintainer can stand up the same proxy before running your commands. Keep the real values for env vars that aren't sensitive, they are often the reason the bug happens, and redact only the secrets: never paste a real API key, virtual key, database URL, or other credential, here or anywhere else in the issue - If the bug applies to more than one of the LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), include proof for every one of them, not just one From c8f385483354a85168d159b90e31c4cd05f58824 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 02:44:47 +0000 Subject: [PATCH 11/23] docs: require a user flow and a stuck-at proof in feature requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/feature_request.yml | 57 ++++++++++++++++++++-- CLAUDE.md | 2 +- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 4cc42901897..d1c31e4ff6d 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -24,10 +24,61 @@ body: validations: required: true - type: textarea - id: motivation + id: user-flow attributes: - label: Motivation, pitch - description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too. + label: User Flow + description: Two numbered lists walking the same end user through the same task, one today without the feature and one with it. Keep the guidance comments in the box while you fill it in, they explain every rule. + value: | + + + Today (without the feature): + + 1. + 2. + 3. + + With the feature: + + 1. + 2. + 3. + validations: + required: true + - type: textarea + id: how-far-you-got + attributes: + label: How far you got + description: Walk the "With the feature" list against a live proxy and paste the commands and output up to the step where you get stuck. Keep the guidance comments in the box while you fill it in, they explain every rule. + value: | + + validations: required: true - type: dropdown diff --git a/CLAUDE.md b/CLAUDE.md index 0e63b86036d..02cc8e024df 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,7 +31,7 @@ When creating PRs, don't set base to `main`. `litellm_internal_staging` is the d When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule -Same applies for filing bug reports and .github/ISSUE_TEMPLATE/bug_report.yml +Same applies for filing bug reports and feature requests, and .github/ISSUE_TEMPLATE/bug_report.yml and .github/ISSUE_TEMPLATE/feature_request.yml If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank From 471bb834ba9962452e446eda34308b9c7bbf7b60 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 02:45:44 +0000 Subject: [PATCH 12/23] docs: rename the feature request flow lists to before/after this feature Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/feature_request.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index d1c31e4ff6d..4ffa491883c 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -27,25 +27,25 @@ body: id: user-flow attributes: label: User Flow - description: Two numbered lists walking the same end user through the same task, one today without the feature and one with it. Keep the guidance comments in the box while you fill it in, they explain every rule. + description: Two numbered lists walking the same end user through the same task, one as it goes today and one as it would ideally go with the feature. Keep the guidance comments in the box while you fill it in, they explain every rule. value: | - - Today (without the feature): + Before this feature (today): 1. 2. 3. - With the feature: + After this feature (ideal user flow): 1. 2. @@ -70,9 +70,9 @@ body: id: how-far-you-got attributes: label: How far you got - description: Walk the "With the feature" list against a live proxy and paste the commands and output up to the step where you get stuck. Keep the guidance comments in the box while you fill it in, they explain every rule. + description: Walk the "After this feature (ideal user flow)" list against a live proxy and paste the commands and output up to the step where you get stuck. Keep the guidance comments in the box while you fill it in, they explain every rule. value: | - + Config / setup the proxy ran with: + + Version or commit: + + Commands and their full output, up to the step that dead-ends: + + What stopped me there: + validations: required: true + - type: checkboxes + id: attempt-attestation + attributes: + label: About that attempt + options: + - label: I ran it against a live proxy myself, with no mocks, and the output above is what I actually got back + required: true - type: dropdown id: component attributes: From 38a1bec5c2de09e722faaf8a498e204d1f24ecd5 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 02:56:38 +0000 Subject: [PATCH 14/23] fix(triage): require evidence of the dead-end in feature requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/scripts/triage_with_llm.py | 5 +++++ tests/test_litellm/test_github_triage_with_llm.py | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/.github/scripts/triage_with_llm.py b/.github/scripts/triage_with_llm.py index a7bd145dbc8..0908c3d76cc 100644 --- a/.github/scripts/triage_with_llm.py +++ b/.github/scripts/triage_with_llm.py @@ -597,6 +597,11 @@ def build_issue_prompt(*, title: str, body: str) -> str: that it does not today). - Motivation / use case with a concrete example (config, API call, UI flow, or scenario showing what's blocked today). + - END-TO-END EVIDENCE OF THE DEAD-END: a video, a screenshot, or the + exact command(s) actually run paired with their real output, + showing the point where the flow stops today. Mocked or stubbed + dependencies do NOT count, and an unfilled template scaffold + (bare headings, empty numbered lists) counts as absent. For an issue that is neither a bug report nor a feature request (a question, support request, or discussion), PASS as long as it has a diff --git a/tests/test_litellm/test_github_triage_with_llm.py b/tests/test_litellm/test_github_triage_with_llm.py index 300fd7c0710..3172dc93e82 100644 --- a/tests/test_litellm/test_github_triage_with_llm.py +++ b/tests/test_litellm/test_github_triage_with_llm.py @@ -678,6 +678,19 @@ class TestBuildPrompts: assert "unfilled template scaffold" in normalized assert "counts as absent, not as evidence" in normalized + def test_issue_feature_rubric_requires_evidence_of_the_dead_end( + self, triage_module + ): + # The feature form asks the requester to walk the ideal flow against a + # live proxy and paste output up to the step that dead-ends, so the + # judge has to demand that evidence, and must not accept an unedited + # scaffold of bare headings as if it were a real attempt. + prompt = triage_module.build_issue_prompt(title="t", body="x") + normalized = " ".join(prompt.split()) + assert "END-TO-END EVIDENCE OF THE DEAD-END" in normalized + assert "showing the point where the flow stops today" in normalized + assert "unfilled template scaffold" in normalized + def test_should_not_crash_when_pr_body_contains_curly_braces(self, triage_module): """User-supplied content with `{` / `}` must NOT be re-parsed by `str.format()`. `format` only scans the template literal for From 1d9500066e31fc2c69012b809f8154978dea197b Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 02:59:10 +0000 Subject: [PATCH 15/23] fix: move feature request guidance out of prefilled values so required means filled Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/feature_request.yml | 52 ++++++++-------------- 1 file changed, 18 insertions(+), 34 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index fc79f5af6d2..5fbed194223 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -27,18 +27,16 @@ body: id: user-flow attributes: label: User Flow - description: Two numbered lists walking the same end user through the same task, one as it goes today and one as it would ideally go with the feature. Keep the guidance comments in the box while you fill it in, they explain every rule. - value: | - - - Before this feature (today): - - 1. - 2. - 3. - - After this feature (ideal user flow): - - 1. - 2. - 3. validations: required: true - type: textarea id: how-far-you-got attributes: label: How far you got - description: Walk the "After this feature (ideal user flow)" list against a live proxy and paste the commands and output up to the step where you get stuck. Keep the guidance comments in the box while you fill it in, they explain every rule. - value: | - + description: | + Run as many steps of the "After this feature (ideal user flow)" list as you can against a live proxy you ran yourself (e.g., `litellm --config config.yaml --detailed_debug` on localhost:4000), then paste the commands (e.g., curl) and their full output, ending at the step that dead-ends. Every rule below applies. + - Say plainly what stopped you there, in user terms: the option you passed came back ignored, the response 400'd naming an unsupported field, there is no button on the page for it. This is what proves the feature is genuinely missing rather than undocumented, and it is the single most useful thing you can give us + - No mocks. Where the flow involves a provider call, hit the real provider API, even though that costs real $. `pytest` commands are not enough + - Include the config.yaml (or SDK setup) and env vars the proxy ran with, plus the version or commit you were on. Keep the real values for env vars that aren't sensitive, and redact only the secrets: never paste a real API key, virtual key, database URL, or other credential, here or anywhere else in the issue + - If the provider already supports this, link their API docs and paste a direct call to them succeeding, so we can see the shape LiteLLM should be sending + - For UI asks: include screenshots of the page you got stuck on and its URL. Scrub keys and tokens out of screenshots too, they show up in headers, request panels, and the Admin UI + placeholder: | Config / setup the proxy ran with: Version or commit: @@ -86,7 +71,6 @@ body: Commands and their full output, up to the step that dead-ends: What stopped me there: - validations: required: true - type: checkboxes From 4eb4511f20267c392b5386d4f37d4a63169fc7f1 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:35:22 -0700 Subject: [PATCH 16/23] chore: make it clearer --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 02cc8e024df..a3c24b84ea8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,7 +31,7 @@ When creating PRs, don't set base to `main`. `litellm_internal_staging` is the d When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule -Same applies for filing bug reports and feature requests, and .github/ISSUE_TEMPLATE/bug_report.yml and .github/ISSUE_TEMPLATE/feature_request.yml +Same applies for filing bug reports and feature requests, with .github/ISSUE_TEMPLATE/bug_report.yml and .github/ISSUE_TEMPLATE/feature_request.yml, respectively If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank From cf1766be1bb49f61c020ed2ba8def74f9be46454 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 16:26:10 +0000 Subject: [PATCH 17/23] chore: drop the feature request attempt attestation and tighten its wording Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/feature_request.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 5fbed194223..30a7e839eb3 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -58,11 +58,11 @@ body: description: | Run as many steps of the "After this feature (ideal user flow)" list as you can against a live proxy you ran yourself (e.g., `litellm --config config.yaml --detailed_debug` on localhost:4000), then paste the commands (e.g., curl) and their full output, ending at the step that dead-ends. Every rule below applies. - - Say plainly what stopped you there, in user terms: the option you passed came back ignored, the response 400'd naming an unsupported field, there is no button on the page for it. This is what proves the feature is genuinely missing rather than undocumented, and it is the single most useful thing you can give us - - No mocks. Where the flow involves a provider call, hit the real provider API, even though that costs real $. `pytest` commands are not enough + - Say plainly what stopped you there, in user terms: the option you passed came back ignored, the response 400'd naming an unsupported field, there is no button on the page for it. This is what proves the feature is genuinely missing rather than undocumented + - No mocks. Where the flow involves a provider call, hit the real provider API, even if it costs real $$$. `pytest` commands are not enough - Include the config.yaml (or SDK setup) and env vars the proxy ran with, plus the version or commit you were on. Keep the real values for env vars that aren't sensitive, and redact only the secrets: never paste a real API key, virtual key, database URL, or other credential, here or anywhere else in the issue - If the provider already supports this, link their API docs and paste a direct call to them succeeding, so we can see the shape LiteLLM should be sending - - For UI asks: include screenshots of the page you got stuck on and its URL. Scrub keys and tokens out of screenshots too, they show up in headers, request panels, and the Admin UI + - For UI asks: include screenshots of the page you got stuck on and its URL. Scrub keys and tokens out of screenshots too (for example, the virtual key is briefly shown in the panel right after you create a virtual key) placeholder: | Config / setup the proxy ran with: @@ -73,13 +73,6 @@ body: What stopped me there: validations: required: true - - type: checkboxes - id: attempt-attestation - attributes: - label: About that attempt - options: - - label: I ran it against a live proxy myself, with no mocks, and the output above is what I actually got back - required: true - type: dropdown id: component attributes: From d4468ba63ab3c2a5048935ad5b5d53327ac684ba Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 16:31:27 +0000 Subject: [PATCH 18/23] chore: say real $ instead of $$$ in the feature request attempt rules Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/feature_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 30a7e839eb3..41b097041f1 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -59,7 +59,7 @@ body: Run as many steps of the "After this feature (ideal user flow)" list as you can against a live proxy you ran yourself (e.g., `litellm --config config.yaml --detailed_debug` on localhost:4000), then paste the commands (e.g., curl) and their full output, ending at the step that dead-ends. Every rule below applies. - Say plainly what stopped you there, in user terms: the option you passed came back ignored, the response 400'd naming an unsupported field, there is no button on the page for it. This is what proves the feature is genuinely missing rather than undocumented - - No mocks. Where the flow involves a provider call, hit the real provider API, even if it costs real $$$. `pytest` commands are not enough + - No mocks. Where the flow involves a provider call, hit the real provider API, even if it costs real $. `pytest` commands are not enough - Include the config.yaml (or SDK setup) and env vars the proxy ran with, plus the version or commit you were on. Keep the real values for env vars that aren't sensitive, and redact only the secrets: never paste a real API key, virtual key, database URL, or other credential, here or anywhere else in the issue - If the provider already supports this, link their API docs and paste a direct call to them succeeding, so we can see the shape LiteLLM should be sending - For UI asks: include screenshots of the page you got stuck on and its URL. Scrub keys and tokens out of screenshots too (for example, the virtual key is briefly shown in the panel right after you create a virtual key) From e59da407503145a2087253f89cb1955d645a309b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:43:16 -0700 Subject: [PATCH 19/23] fix(triage): ask for dead-end evidence in feature request recovery comments --- .github/scripts/triage_with_llm.py | 13 +++++++++---- .../test_litellm/test_github_triage_with_llm.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/scripts/triage_with_llm.py b/.github/scripts/triage_with_llm.py index 0908c3d76cc..6499a063b39 100644 --- a/.github/scripts/triage_with_llm.py +++ b/.github/scripts/triage_with_llm.py @@ -843,8 +843,11 @@ def format_issue_close_comment(verdict: dict) -> str: "video, a screenshot, or the exact commands you ran with their real output / " "traceback) plus expected vs. actual behavior. Written steps with no run output, " "video, or screenshot don't count, and mocked or stubbed runs don't count.\n" - " - For **feature requests**: a concrete description of what should change, plus a " - "use case and example (config / API call / UI flow).\n" + " - For **feature requests**: a concrete description of what should change, a " + "use case and example (config / API call / UI flow), plus end-to-end evidence of " + "the dead-end (a video, a screenshot, or the exact commands you ran with their " + "real output showing where the flow stops today). Mocked or stubbed runs don't " + "count.\n" "2. Comment `@agent-shin reconsider`. I'll re-run triage and reopen the issue if it " "now meets the bar. (GitHub doesn't let external authors reopen an issue a maintainer " "or bot closed, so the comment-based reconsider is the reliable path.)\n" @@ -950,8 +953,10 @@ def format_grace_warning_issue_comment(verdict: dict) -> str: "screenshot, or the exact commands you ran with their real output / traceback) plus " "expected vs. actual behavior. Written steps with no run output don't count, and " "mocked or stubbed runs don't count.\n" - "- For **feature requests**: a concrete description of what should change, plus a use " - "case and example (config / API call / UI flow).\n" + "- For **feature requests**: a concrete description of what should change, a use " + "case and example (config / API call / UI flow), plus end-to-end evidence of the " + "dead-end (a video, a screenshot, or the exact commands you ran with their real " + "output showing where the flow stops today). Mocked or stubbed runs don't count.\n" "\n" "**If the issue does get auto-closed in 2 hours**, comment `@agent-shin reconsider` " "and I'll re-evaluate. If it now meets the bar, I'll reopen the issue.\n" diff --git a/tests/test_litellm/test_github_triage_with_llm.py b/tests/test_litellm/test_github_triage_with_llm.py index 3172dc93e82..50b198f62c5 100644 --- a/tests/test_litellm/test_github_triage_with_llm.py +++ b/tests/test_litellm/test_github_triage_with_llm.py @@ -207,6 +207,23 @@ class TestCloseCommentText: assert "end-to-end qa proof" in body.lower() assert "mock" in body.lower() + def test_issue_recovery_comments_should_name_feature_dead_end_evidence( + self, triage_module + ): + # The feature-request pass bar demands end-to-end evidence of the + # dead-end, so the close and grace-warning recovery bullets must ask + # for it too — otherwise a requester follows those exact instructions + # (description + use case only) and fails `reconsider` again with no + # hint of what else was needed. + verdict = {"verdict": "fail", "missing": [], "explanation": ""} + for body in ( + triage_module.format_issue_close_comment(verdict), + triage_module.format_grace_warning_issue_comment(verdict), + ): + normalized = " ".join(body.split()) + assert "end-to-end evidence of the dead-end" in normalized + assert "showing where the flow stops today" in normalized + def test_all_agent_shin_comments_should_use_bullet_train_emoji(self, triage_module): # The bullet train (🚅) is Agent Shin's symbol, matching the LiteLLM # logo; the previous wave (👋) was generic and didn't match the bot's From 812abcc7f55153cf5ac11a49d76c0b3a6775f4e3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:57:46 -0700 Subject: [PATCH 20/23] fix(triage): track and credit feature dead-end evidence in the verdict --- .github/scripts/triage_with_llm.py | 17 ++++++++---- .../test_github_triage_with_llm.py | 26 +++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/.github/scripts/triage_with_llm.py b/.github/scripts/triage_with_llm.py index 6499a063b39..e23a012425a 100644 --- a/.github/scripts/triage_with_llm.py +++ b/.github/scripts/triage_with_llm.py @@ -597,11 +597,13 @@ def build_issue_prompt(*, title: str, body: str) -> str: that it does not today). - Motivation / use case with a concrete example (config, API call, UI flow, or scenario showing what's blocked today). - - END-TO-END EVIDENCE OF THE DEAD-END: a video, a screenshot, or the - exact command(s) actually run paired with their real output, - showing the point where the flow stops today. Mocked or stubbed - dependencies do NOT count, and an unfilled template scaffold - (bare headings, empty numbered lists) counts as absent. + - END-TO-END EVIDENCE OF THE DEAD-END (set + `has_dead_end_evidence=true` only when this is present): a video, + a screenshot, or the exact command(s) actually run paired with + their real output, showing the point where the flow stops today. + Mocked or stubbed dependencies do NOT count, and an unfilled + template scaffold (bare headings, empty numbered lists) counts as + absent. For an issue that is neither a bug report nor a feature request (a question, support request, or discussion), PASS as long as it has a @@ -615,6 +617,7 @@ def build_issue_prompt(*, title: str, body: str) -> str: "has_repro": boolean, "has_expected_vs_actual": boolean, "has_motivation_example": boolean, + "has_dead_end_evidence": boolean, "missing": ["plain-english strings naming what is missing"], "explanation": "1-2 sentence reasoning for the team to skim" }} @@ -712,6 +715,10 @@ _ISSUE_BUG_LABELS: tuple[tuple[str, str], ...] = ( ) _ISSUE_FEATURE_LABELS: tuple[tuple[str, str], ...] = ( ("has_motivation_example", "Motivation and concrete example"), + ( + "has_dead_end_evidence", + "End-to-end evidence of the dead-end (video, screenshot, or command + real output)", + ), ) diff --git a/tests/test_litellm/test_github_triage_with_llm.py b/tests/test_litellm/test_github_triage_with_llm.py index 50b198f62c5..96b77e80457 100644 --- a/tests/test_litellm/test_github_triage_with_llm.py +++ b/tests/test_litellm/test_github_triage_with_llm.py @@ -306,6 +306,27 @@ class TestCloseCommentText: assert "Expected vs. actual behavior" in body assert "- ✅ End-to-end evidence of the bug" not in body + def test_issue_close_comment_should_credit_feature_dead_end_evidence( + self, triage_module + ): + # A feature requester who pasted their dead-end run but skipped the + # motivation must see the evidence credited and only the motivation + # listed as a gap — without a dedicated verdict field the praise + # block could never acknowledge the work they did do. + body = triage_module.format_issue_close_comment( + { + "verdict": "fail", + "kind": "feature", + "has_motivation_example": False, + "has_dead_end_evidence": True, + "missing": ["motivation / use case"], + "explanation": "no use case given", + } + ) + assert "What you got right" in body + assert "- ✅ End-to-end evidence of the dead-end" in body + assert "- ✅ Motivation and concrete example" not in body + def test_close_comments_should_use_softer_park_for_later_framing( self, triage_module ): @@ -707,6 +728,11 @@ class TestBuildPrompts: assert "END-TO-END EVIDENCE OF THE DEAD-END" in normalized assert "showing the point where the flow stops today" in normalized assert "unfilled template scaffold" in normalized + # The evidence has its own verdict field so feature requesters who + # provided it get credited in "What you got right", exactly like + # `has_repro` credits bug evidence. + assert "`has_dead_end_evidence=true` only when this is present" in normalized + assert '"has_dead_end_evidence": boolean' in normalized def test_should_not_crash_when_pr_body_contains_curly_braces(self, triage_module): """User-supplied content with `{` / `}` must NOT be re-parsed by From b144b15d48ddf78e9096488bad531aee8709dfd3 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 11 Aug 2026 11:02:57 -0700 Subject: [PATCH 21/23] fix(proxy): add config_updated_at audit timestamp for virtual keys (#36488) * fix(proxy): add config_updated_at audit timestamp for virtual keys updated_at carries Prisma's @updatedAt, so every batched spend flush rewrites it and it cannot distinguish config changes from usage. Add an additive config_updated_at column stamped only by key management writes (update, bulk update, regenerate, block, unblock) via a shared helper, expose it on key responses, and switch the key page's Last Updated to it with a created_at fallback. * test(proxy): assert config_updated_at survives key archival * refactor(proxy): rename config_updated_at to settings_updated_at --- .../migration.sql | 5 + .../litellm_proxy_extras/schema.prisma | 2 + litellm/models/verification_token.py | 1 + .../key_management_endpoints.py | 7 +- .../management_helpers/key_settings_audit.py | 14 +++ litellm/proxy/schema.prisma | 2 + litellm/proxy/utils.py | 3 +- schema.prisma | 2 + .../proxy/db/test_db_spend_update_writer.py | 64 ++++++++++ .../test_key_management_endpoints.py | 110 ++++++++++++++++-- tests/test_litellm/proxy/test_proxy_utils.py | 22 ++++ .../components/key_team_helpers/key_list.tsx | 1 + .../templates/key_info_view.test.tsx | 40 +++++++ .../components/templates/key_info_view.tsx | 4 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 + 15 files changed, 269 insertions(+), 14 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260810000000_add_verificationtoken_settings_updated_at/migration.sql create mode 100644 litellm/proxy/management_helpers/key_settings_audit.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260810000000_add_verificationtoken_settings_updated_at/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260810000000_add_verificationtoken_settings_updated_at/migration.sql new file mode 100644 index 00000000000..fa12f4eb138 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260810000000_add_verificationtoken_settings_updated_at/migration.sql @@ -0,0 +1,5 @@ +-- AlterTable +ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "settings_updated_at" TIMESTAMP(3); + +-- AlterTable +ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN IF NOT EXISTS "settings_updated_at" TIMESTAMP(3); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 33fd9389b63..854602f5380 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -452,6 +452,7 @@ model LiteLLM_VerificationToken { created_by String? updated_at DateTime? @default(now()) @updatedAt @map("updated_at") updated_by String? + settings_updated_at DateTime? @map("settings_updated_at") last_active DateTime? // When this key was last used rotation_count Int? @default(0) // Number of times key has been rotated auto_rotate Boolean? @default(false) // Whether this key should be auto-rotated @@ -548,6 +549,7 @@ model LiteLLM_DeletedVerificationToken { created_by String? // Original creator updated_at DateTime? // Last update timestamp before deletion updated_by String? // Last user who updated before deletion + settings_updated_at DateTime? // Last configuration change before deletion last_active DateTime? // When this key was last used before deletion rotation_count Int? @default(0) auto_rotate Boolean? @default(false) diff --git a/litellm/models/verification_token.py b/litellm/models/verification_token.py index ea822c2dab0..fec3caec457 100644 --- a/litellm/models/verification_token.py +++ b/litellm/models/verification_token.py @@ -49,6 +49,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase): created_by: str | None = None updated_at: datetime | None = None updated_by: str | None = None + settings_updated_at: datetime | None = None last_active: datetime | None = None object_permission_id: str | None = None object_permission: LiteLLM_ObjectPermissionTable | None = None diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 38b5d755535..836b223c6ad 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -88,6 +88,7 @@ from litellm.proxy.management_endpoints.common_utils import ( from litellm.proxy.management_endpoints.model_management_endpoints import ( _add_model_to_db, ) +from litellm.proxy.management_helpers.key_settings_audit import with_settings_updated_at from litellm.proxy.management_helpers.object_permission_utils import ( _set_object_permission, attach_object_permission_to_dict, @@ -4693,7 +4694,7 @@ async def _execute_virtual_key_regeneration( updated_token: Final = await VerificationTokenRepository(prisma_client).table.update( where={"token": hashed_api_key}, - data=jsonified_update_data, + data=with_settings_updated_at(jsonified_update_data), ) updated_token_dict: Final[dict[str, object]] = dict(updated_token) if updated_token is not None else {} updated_token_dict["key"] = new_token @@ -6203,7 +6204,7 @@ async def block_key( record: Final = await _prisma_table(VerificationTokenRepository(prisma_client)).update( where={"token": hashed_token}, - data={"blocked": True}, + data=with_settings_updated_at({"blocked": True}), ) ## UPDATE KEY CACHE - invalidate so next read re-fetches from DB @@ -6316,7 +6317,7 @@ async def unblock_key( record: Final = await _prisma_table(VerificationTokenRepository(prisma_client)).update( where={"token": hashed_token}, - data={"blocked": False}, + data=with_settings_updated_at({"blocked": False}), ) ## UPDATE KEY CACHE - invalidate so next read re-fetches from DB diff --git a/litellm/proxy/management_helpers/key_settings_audit.py b/litellm/proxy/management_helpers/key_settings_audit.py new file mode 100644 index 00000000000..a2c4bd8cac0 --- /dev/null +++ b/litellm/proxy/management_helpers/key_settings_audit.py @@ -0,0 +1,14 @@ +"""Audit stamping for virtual key configuration changes.""" + +from collections.abc import Mapping +from datetime import datetime, timezone + + +def with_settings_updated_at(data: Mapping[str, object]) -> dict[str, object]: + """Stamp a key update payload with the time its configuration changed. + + ``updated_at`` carries Prisma's ``@updatedAt`` and so is rewritten by every + spend flush, which makes it useless for auditing; ``settings_updated_at`` is + written only from key-management write paths. + """ + return {**data, "settings_updated_at": datetime.now(timezone.utc)} diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 33fd9389b63..854602f5380 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -452,6 +452,7 @@ model LiteLLM_VerificationToken { created_by String? updated_at DateTime? @default(now()) @updatedAt @map("updated_at") updated_by String? + settings_updated_at DateTime? @map("settings_updated_at") last_active DateTime? // When this key was last used rotation_count Int? @default(0) // Number of times key has been rotated auto_rotate Boolean? @default(false) // Whether this key should be auto-rotated @@ -548,6 +549,7 @@ model LiteLLM_DeletedVerificationToken { created_by String? // Original creator updated_at DateTime? // Last update timestamp before deletion updated_by String? // Last user who updated before deletion + settings_updated_at DateTime? // Last configuration change before deletion last_active DateTime? // When this key was last used before deletion rotation_count Int? @default(0) auto_rotate Boolean? @default(false) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index dd0c57aa911..b5972935806 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -135,6 +135,7 @@ from litellm.proxy.hooks.sensitive_data_routing import ( _PROXY_SensitiveDataRoutingHandler, ) from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +from litellm.proxy.management_helpers.key_settings_audit import with_settings_updated_at from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.config_repository import ConfigRepository @@ -3996,7 +3997,7 @@ class PrismaClient: db_data["token"] = token response: Final = await VerificationTokenRepository(self).table.update( where={"token": token}, - data={**db_data}, + data=with_settings_updated_at(db_data), ) verbose_proxy_logger.debug("\033[91m" + f"DB Token Table update succeeded {response}" + "\033[0m") _data: dict = {} diff --git a/schema.prisma b/schema.prisma index 33fd9389b63..854602f5380 100644 --- a/schema.prisma +++ b/schema.prisma @@ -452,6 +452,7 @@ model LiteLLM_VerificationToken { created_by String? updated_at DateTime? @default(now()) @updatedAt @map("updated_at") updated_by String? + settings_updated_at DateTime? @map("settings_updated_at") last_active DateTime? // When this key was last used rotation_count Int? @default(0) // Number of times key has been rotated auto_rotate Boolean? @default(false) // Whether this key should be auto-rotated @@ -548,6 +549,7 @@ model LiteLLM_DeletedVerificationToken { created_by String? // Original creator updated_at DateTime? // Last update timestamp before deletion updated_by String? // Last user who updated before deletion + settings_updated_at DateTime? // Last configuration change before deletion last_active DateTime? // When this key was last used before deletion rotation_count Int? @default(0) auto_rotate Boolean? @default(false) diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index 51810a28cdd..2c426e0f071 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -2157,3 +2157,67 @@ async def test_daily_transaction_compression_saved_tokens_zero_when_absent(): assert transaction["compression_saved_tokens"] == 0 assert transaction["compression_savings_spend"] == 0 assert transaction["prompt_caching_savings_spend"] == 0 + + +@pytest.mark.asyncio +async def test_commit_spend_updates_to_db_does_not_stamp_key_settings_updated_at(): + """Spend flushes must leave settings_updated_at alone, or it decays into + another `updated_at` and stops being an audit signal.""" + db_writer = DBSpendUpdateWriter() + + mock_batcher = MagicMock() + mock_batcher.litellm_verificationtoken = MagicMock() + mock_batcher.litellm_verificationtoken.update_many = MagicMock() + mock_batcher.litellm_usertable = MagicMock() + mock_batcher.litellm_usertable.update_many = MagicMock() + mock_batcher.litellm_teamtable = MagicMock() + mock_batcher.litellm_teamtable.update_many = MagicMock() + mock_batcher.litellm_teammembership = MagicMock() + mock_batcher.litellm_teammembership.update_many = MagicMock() + mock_batcher.litellm_organizationtable = MagicMock() + mock_batcher.litellm_organizationtable.update_many = MagicMock() + mock_batcher.litellm_tagtable = MagicMock() + mock_batcher.litellm_tagtable.update_many = MagicMock() + mock_batcher.litellm_agentstable = MagicMock() + mock_batcher.litellm_agentstable.update_many = MagicMock() + + mock_transaction = AsyncMock() + mock_transaction.__aenter__ = AsyncMock(return_value=mock_transaction) + mock_transaction.__aexit__ = AsyncMock(return_value=False) + mock_transaction.batch_ = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=mock_batcher), + __aexit__=AsyncMock(return_value=False), + ) + ) + + mock_prisma_client = MagicMock() + mock_prisma_client.db = MagicMock() + mock_prisma_client.db.tx = MagicMock(return_value=mock_transaction) + + token = "hashed-token-abc" + response_cost = 0.25 + db_spend_update_transactions = { + "user_list_transactions": {}, + "end_user_list_transactions": {}, + "key_list_transactions": {token: response_cost}, + "team_list_transactions": {}, + "team_member_list_transactions": {}, + "org_list_transactions": {}, + "tag_list_transactions": {}, + "agent_list_transactions": {}, + } + + with patch("litellm.proxy.utils._raise_failed_update_spend_exception"): + await db_writer._commit_spend_updates_to_db( + prisma_client=mock_prisma_client, + n_retry_times=0, + proxy_logging_obj=MagicMock(), + db_spend_update_transactions=db_spend_update_transactions, + ) + + mock_batcher.litellm_verificationtoken.update_many.assert_called_once() + call_kwargs = mock_batcher.litellm_verificationtoken.update_many.call_args[1] + assert call_kwargs["where"] == {"token": token} + assert set(call_kwargs["data"]) == {"spend", "last_active"} + assert call_kwargs["data"]["spend"] == {"increment": response_cost} diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 8f151ed882c..4c13a3367d9 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -2293,9 +2293,9 @@ async def test_unblock_key_supports_both_sk_and_hashed_tokens(monkeypatch): ) # Verify that the database update was called with hashed token - mock_prisma_client.db.litellm_verificationtoken.update.assert_called_with( - where={"token": test_hashed_token}, data={"blocked": False} - ) + sk_token_call = mock_prisma_client.db.litellm_verificationtoken.update.call_args.kwargs + assert sk_token_call["where"] == {"token": test_hashed_token} + assert sk_token_call["data"]["blocked"] is False assert result == mock_key_record @@ -2313,9 +2313,9 @@ async def test_unblock_key_supports_both_sk_and_hashed_tokens(monkeypatch): ) # Verify that the database update was called with the same hashed token - mock_prisma_client.db.litellm_verificationtoken.update.assert_called_with( - where={"token": test_hashed_token}, data={"blocked": False} - ) + hashed_token_call = mock_prisma_client.db.litellm_verificationtoken.update.call_args.kwargs + assert hashed_token_call["where"] == {"token": test_hashed_token} + assert hashed_token_call["data"]["blocked"] is False assert result == mock_key_record @@ -2849,9 +2849,10 @@ async def test_block_key_existing_key_succeeds(monkeypatch): mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once_with( where={"token": test_hashed_token} ) - mock_prisma_client.db.litellm_verificationtoken.update.assert_called_once_with( - where={"token": test_hashed_token}, data={"blocked": True} - ) + mock_prisma_client.db.litellm_verificationtoken.update.assert_called_once() + block_call = mock_prisma_client.db.litellm_verificationtoken.update.call_args.kwargs + assert block_call["where"] == {"token": test_hashed_token} + assert block_call["data"]["blocked"] is True assert result == mock_updated_record @@ -4717,6 +4718,7 @@ def test_transform_verification_tokens_to_deleted_records(): user_role=LitellmUserRoles.PROXY_ADMIN.value, ) + config_stamp = datetime(2026, 8, 10, 12, 30, 45, tzinfo=timezone.utc) key1 = LiteLLM_VerificationToken( token="hashed-token-1", user_id="user-123", @@ -4733,6 +4735,7 @@ def test_transform_verification_tokens_to_deleted_records(): model_spend={}, soft_budget_cooldown=False, allowed_routes=[], + settings_updated_at=config_stamp, ) key2 = LiteLLM_VerificationToken( @@ -4775,6 +4778,7 @@ def test_transform_verification_tokens_to_deleted_records(): assert record1["token"] == "hashed-token-1" assert record1["user_id"] == "user-123" assert record1["team_id"] == "team-456" + assert record1["settings_updated_at"] == config_stamp assert isinstance(record1["aliases"], str) assert isinstance(record1["config"], str) assert isinstance(record1["permissions"], str) @@ -15817,3 +15821,91 @@ async def test_regenerate_key_output_token_estimate_lowered_rejected_for_non_adm assert exc.value.status_code == 403 assert "Only proxy admins can set" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_execute_virtual_key_regeneration_stamps_settings_updated_at(): + """Regenerate rewrites the key's config, so it must move settings_updated_at.""" + from datetime import datetime, timezone + + from litellm.proxy._types import RegenerateKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _execute_virtual_key_regeneration, + ) + + mock_prisma_client = _make_regenerate_mock_prisma() + + with _patch_regenerate_side_effects(): + before = datetime.now(timezone.utc) + await _execute_virtual_key_regeneration( + prisma_client=mock_prisma_client, + key_in_db=_make_regenerate_existing_key(), + hashed_api_key="abc123", + key="abc123", + data=RegenerateKeyRequest(max_budget=42.0), + user_api_key_dict=_make_regenerate_user_api_key_dict(), + litellm_changed_by=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + ) + after = datetime.now(timezone.utc) + + sent = mock_prisma_client.db.litellm_verificationtoken.update.call_args.kwargs["data"] + assert sent["max_budget"] == 42.0 + assert before <= sent["settings_updated_at"] <= after + + +@pytest.mark.asyncio +async def test_block_key_stamps_settings_updated_at(monkeypatch): + """Blocking a key is a config change, not spend activity.""" + from datetime import datetime, timezone + + from litellm.proxy._types import BlockKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import block_key + + mock_prisma_client, _ = _setup_block_unblock_mocks(monkeypatch) + + before = datetime.now(timezone.utc) + await block_key( + data=BlockKeyRequest(key="sk-test123456789"), + http_request=MagicMock(), + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin_user", + ), + litellm_changed_by=None, + ) + after = datetime.now(timezone.utc) + + sent = mock_prisma_client.db.litellm_verificationtoken.update.call_args.kwargs["data"] + assert sent["blocked"] is True + assert before <= sent["settings_updated_at"] <= after + + +@pytest.mark.asyncio +async def test_unblock_key_stamps_settings_updated_at(monkeypatch): + """Unblocking a key is a config change, not spend activity.""" + from datetime import datetime, timezone + + from litellm.proxy._types import BlockKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import unblock_key + + mock_prisma_client, _ = _setup_block_unblock_mocks(monkeypatch) + + before = datetime.now(timezone.utc) + await unblock_key( + data=BlockKeyRequest(key="sk-test123456789"), + http_request=MagicMock(), + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin_user", + ), + litellm_changed_by=None, + ) + after = datetime.now(timezone.utc) + + sent = mock_prisma_client.db.litellm_verificationtoken.update.call_args.kwargs["data"] + assert sent["blocked"] is False + assert before <= sent["settings_updated_at"] <= after diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index a8e81e92ebd..a4f93e90673 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -1169,3 +1169,25 @@ async def test_prisma_health_check_failure_redacts_database_credentials(caplog): assert emitted assert all("hunter2" not in message for message in emitted) assert any("postgresql://REDACTED@db.internal" in message for message in emitted) + + +@pytest.mark.asyncio +async def test_update_data_key_branch_stamps_settings_updated_at(): + """`updated_at` carries Prisma's @updatedAt and is rewritten by every spend + flush, so key config edits need their own audit column.""" + from datetime import datetime, timezone + from unittest.mock import AsyncMock + + from litellm.proxy.utils import PrismaClient + + client = MagicMock() + client.jsonify_object = MagicMock(side_effect=lambda data: dict(data)) + client.db.litellm_verificationtoken.update = AsyncMock(return_value=None) + + before = datetime.now(timezone.utc) + await PrismaClient.update_data(client, token="sk-test-key", data={"models": ["gpt-4"]}) + after = datetime.now(timezone.utc) + + sent = client.db.litellm_verificationtoken.update.call_args.kwargs["data"] + assert sent["models"] == ["gpt-4"] + assert before <= sent["settings_updated_at"] <= after diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx index 4b446b0c283..920d1f4af5a 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx @@ -60,6 +60,7 @@ export interface KeyResponse { created_at: string; created_by?: string; updated_at: string; + settings_updated_at?: string | null; last_active: string | null; team_spend: number; team_alias: string; diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx index 8e9dd2fa975..680fbe9ff1e 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx @@ -149,6 +149,46 @@ describe("KeyInfoView", () => { await userEvent.click(await screen.findByRole("button", { name: /more key actions/i })); }; + describe("last updated", () => { + const renderWithTimestamps = (overrides: Partial) => { + vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock); + + return renderWithProviders( + {}} + keyId={"test-key-id"} + onKeyDataUpdate={() => {}} + teams={[]} + />, + ); + }; + + const findLastUpdatedText = async () => { + const label = await screen.findByText("Last Updated"); + return label.closest("div")?.parentElement?.parentElement?.textContent ?? ""; + }; + + it("should show when the key was last configured, not when it last recorded spend", async () => { + renderWithTimestamps({ settings_updated_at: "2022-06-15T12:00:00Z" }); + + expect(await findLastUpdatedText()).toMatch(/Jun \d+, 2022/); + expect(screen.queryByText(/Jun \d+, 2023/)).not.toBeInTheDocument(); + }); + + it("should fall back to creation time for a key that was never reconfigured", async () => { + renderWithTimestamps({ settings_updated_at: null }); + + expect(await findLastUpdatedText()).toMatch(/Jun \d+, 2021/); + expect(screen.queryByText(/Jun \d+, 2023/)).not.toBeInTheDocument(); + }); + }); + it("should render tags", async () => { vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock); diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index b1ada186bb8..6a2a4a88425 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -444,6 +444,8 @@ export default function KeyInfoView({ ); }; + const lastConfiguredAt = currentKeyData.settings_updated_at || currentKeyData.created_at; + const parentTeam = currentKeyData.team_id ? teamsData?.find((team) => team.team_id === currentKeyData.team_id) : null; const budgetDisplay = @@ -468,7 +470,7 @@ export default function KeyInfoView({ currentKeyData.created_by || "", createdAt: currentKeyData.created_at ? formatTimestamp(currentKeyData.created_at) : "", - lastUpdated: currentKeyData.updated_at ? formatTimestamp(currentKeyData.updated_at) : "", + lastUpdated: lastConfiguredAt ? formatTimestamp(lastConfiguredAt) : "", lastActive: currentKeyData.last_active ? formatTimestamp(currentKeyData.last_active) : "Never", expires: currentKeyData.expires ? formatTimestamp(currentKeyData.expires) : "Never", }} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 8989e098f54..35adc9e92cf 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -26280,6 +26280,8 @@ export interface components { } | null; /** Rpm Limit */ rpm_limit?: number | null; + /** Settings Updated At */ + settings_updated_at?: string | null; /** * Soft Budget Cooldown * @default false @@ -27719,6 +27721,8 @@ export interface components { } | null; /** Rpm Limit */ rpm_limit?: number | null; + /** Settings Updated At */ + settings_updated_at?: string | null; /** * Soft Budget Cooldown * @default false @@ -34887,6 +34891,8 @@ export interface components { rpm_limit_per_model?: { [key: string]: number; } | null; + /** Settings Updated At */ + settings_updated_at?: string | null; /** Soft Budget */ soft_budget?: number | null; /** From 84d6666a594e7ddbf95ba51db8c74f547f5f38ee Mon Sep 17 00:00:00 2001 From: Deepanshu Lulla Date: Tue, 11 Aug 2026 14:49:18 -0400 Subject: [PATCH 22/23] feat(router): add required-AND (&) tag prefix and allow_fail_open flag (#36193) * feat(router): add required-AND (&) tag prefix and allow_fail_open flag Tag routing supported inclusion-OR and independent "!" negation, but had no way to express a hard "must match all of these" constraint per request, and no way for a model group to opt into degrading gracefully instead of raising when a constraint eliminates every deployment. Adds a "&tag" prefix for required-AND inclusion, composing with existing plain (OR) and "!" (negate) tags: negation still applies first, then required tags narrow the survivors, then plain tags apply today's OR/AND preference logic unchanged. Adds model_info.allow_fail_open (default false) so a chain can opt into falling back to the default-tagged pool instead of raising no_deployments_with_tag_routing when "!" or "&" empties the candidate set; existing chains without the flag keep today's fail-closed behavior exactly. * fix(router): gate mixed negation on allow_fail_open and stop diluting required-only requests Two gaps in the initial required-AND/allow_fail_open change: a "!" exclusion combined with a plain positive tag that emptied the candidate set raised unconditionally, bypassing allow_fail_open entirely, since the fail-open check only looked at required-AND exhaustion. And a request using only "&" tags could get narrowed down to just the deployment matching an incidental tag_regex/User-Agent preference, silently dropping other deployments that satisfied the required tags but had no tag_regex at all. Fixes both: the fail-open check now fires whenever either "!" or "&" leaves the candidate set empty, not just "&". And regex/header preference no longer counts as a positive filter when a required-AND ask is present, so a required-only request returns every deployment satisfying the required tags regardless of regex/header matching. Also regenerates ui/litellm-dashboard/src/lib/http/schema.d.ts for the new model_info.allow_fail_open field, and removes source comments explaining the router logic per repository convention. * fix(router): let allow_fail_open cover a non-empty !/& survivor set that fails the plain-tag preference The unconditional raise inside the has_positive_filter loop was the one remaining path a chain could hit despite setting allow_fail_open: when "!" or "&" leaves a non-empty candidate set but none of the survivors match the request's plain preference tag or carry "default", the request still raised instead of degrading. Routes that raise through the same allow_fail_open check used everywhere else, so it now falls back to the default-tagged pool for opted-in chains and keeps raising unconditionally for everyone else. This also let the now-redundant pre-loop empty-candidates shortcut be removed, since the loop reaches the same outcome on its own. * fix(router): deny allow_fail_open when an unrecognized required tag is masking a satisfiable answer A caller could add a single "&" tag no deployment in the group has ever carried to force an empty required-AND set on demand. On a chain with allow_fail_open, that emptied set fell back to the default-tagged pool unconditionally, discarding every other constraint merged into the same request, including ones inherited from key/team policy, even when the rest of those constraints were still individually satisfiable. Before falling back, drop any required tag not carried by any deployment in the group and recompute: if a specific, non-empty answer exists using only the recognized tags, the unrecognized tag was the actual cause of the exhaustion, and fail-open must not paper over it. If every required tag is already recognized, or none are, there's nothing hidden behind an invented tag, and fail-open proceeds exactly as before; this keeps a single opted-in deployment's legitimate catch-all behavior working when a caller's tag simply doesn't exist anywhere in that group. Ratchets ANN401 and LIT001 budgets down to reflect fixes already earned in this branch. * test(router): cover required-AND, allow_fail_open, and unknown-tag denial across fallback chains and model groups Extends coverage beyond single-hop scenarios: & exhausting a primary group falls through to a fallback group exactly like ! already does; !, &, and allow_fail_open composed together across three chained model groups each raise or fall back independently per-hop; and the unknown-tag denial from the previous commit is evaluated fresh per hop rather than leaking state across groups in a fallback chain. * feat(router): add model_info.enable_tag_filtering per-model-group override enable_tag_filtering was router-wide only: an operator turning it on for one model group that needs tag-driven routing exposed every other model group on the same proxy to the same tag evaluation, even ones that never use tags. Adds model_info.enable_tag_filtering, checked against any deployment sharing a model_name, so a chain can flip the router-wide default in either direction for itself alone: opt a specific group into filtering while the rest of the proxy stays off, or opt a group out (e.g. an incident-response catch-all) while the rest of the proxy enforces it. Precedence, low to high: router-wide default, then the chain override if set, then the existing request-level escalation (from key/team settings), which still only ever turns filtering on, never off, over whatever the router and chain already decided. Also regenerates ui/litellm-dashboard/src/lib/http/schema.d.ts for the new field. * fix(router): gate plain-tag exhaustion on allow_fail_open when the tag is known to the group A model group where every deployment is tagged "default" (a legitimate cross-cutting safety-net pattern) never has an empty default_deployments list, so the existing exhaustion check (len(new)==0 and len(default)==0) never fired for a plain positive tag that matched nothing among the currently healthy candidates. The request silently fell through to whatever "default"-tagged deployment happened to survive, even when allow_fail_open was never set and the caller's intent (e.g. quality:high) was never honored. Adds a check for whether the requested tag is part of the group's real vocabulary at all: if some deployment configured under this model_name (regardless of current health) genuinely carries the tag, and nothing healthy currently matches it, the request now raises by default or falls back per allow_fail_open, through the same _resolve_or_fail_open gate every other exhaustion path already uses. A tag that's foreign to the group entirely (e.g. one meant for an unrelated mechanism sharing the same request-tags list) keeps falling back to the default pool unconditionally, unchanged, since there's nothing this group's own routing intent could be violating. * fix(router): preserve inherited tag constraints when allow_fail_open discards a caller-caused exhaustion Adds metadata.caller_tags in litellm_pre_call_utils.py, populated only from what the request itself supplied (header, body tags, body metadata.tags), never from key/team metadata merged into the same metadata.tags list. get_deployments_for_tag now uses it to compute a trusted-only pool before falling open: a required/excluded tag attributable to the caller can be discarded on fail-open, one inherited from key/team policy cannot. If the trusted-only pool is itself empty, allow_fail_open raises instead of silently routing around an unsatisfiable inherited constraint. When caller_tags carries no information at all (direct SDK Router usage, bypassing the proxy layer), behavior is unchanged: unconditional fall-open to the default pool, exactly as before this fix. * feat(router): add opt-in tag_routing_prefix for collision-proof tag disambiguation router_settings.tag_routing_prefix lets a caller explicitly mark which x-litellm-tags/metadata.tags values are routing directives, exempting them from the known-tag-vocabulary heuristic used to guard fail-open against caller-invented "&"/"!" tags. Unprefixed tags keep going through today's existing handling unchanged (hybrid, no migration required); default "" is a full no-op. Fixes a bug caught during live-proxy verification: the prefix-stripped "confirmed" set kept the "&"/"!" marker character, so it never matched required_set/excluded_set (which _split_tags always strips bare) -- the entire trusted-required/excluded-tag mechanism silently no-opped for its primary use case. Adds regression tests for the bare-value mismatch and updates existing _chain_allows_fail_open/_tag_known_to_group/ _caller_constraint_sets call sites for the new routing_confirmed/ routing_prefix parameters. * fix(router): resolve model_info.enable_tag_filtering override from the full model group, not just healthy deployments Cooldown filtering runs before get_deployments_for_tag, so _chain_tag_filtering_override only saw the survivors of that filter. A model group whose only enable_tag_filtering-carrying deployment goes into cooldown lost the override entirely, silently falling back to the router-wide default and letting any !/&/tag constraint on that chain be bypassed by driving the one overriding deployment into cooldown. Resolve the override from every deployment configured for the model instead, mirroring _tag_known_to_group's existing pattern. Verified live: with a bad-key deployment carrying the override forced into real cooldown via allowed_fails=1, an explicit "!provider:openai" ban on the remaining deployment reproducibly returned 200 via OpenAI before this fix and 401 (tag filtering still enforced) after it. * fix(router): avoid Final-reassignment lint error and a MagicMock router fixture gap from tag_routing_prefix _chain_tag_filtering_override's try/except reassigned a Final-annotated name across branches, which basedpyright flags as illegal; extracted the lookup-with-fallback into its own helper so the binding is assigned once. Also sets tag_routing_prefix on the bare MagicMock router used by test_router_tag_regex_routing.py's fixture, which otherwise returns an auto-generated MagicMock (truthy, non-string) for the new attribute and crashes _strip_routing_prefix's removeprefix() call. * fix(router): key inherited-tag protection off provenance, not value subtraction allow_fail_open's trusted-only pool computed "not caller-attributable" as required_set - caller_required_set. A caller who resubmits the exact value of an inherited "&"/"!" tag (e.g. an inherited "®ion:eu" alongside a caller-supplied "®ion:eu" plus a conflicting "!region:eu") collapses both origins to the same set value, so the subtraction zeroes out the inherited requirement's protection too, letting fail-open route outside a key/team-enforced constraint. Adds metadata.inherited_tags in litellm_pre_call_utils.py: a snapshot of "tags" taken after key/team/project policy is merged in but before this request's own caller-supplied tags are merged on top. A required or excluded tag is now protected from fail-open discard if it has ANY inherited backing (set intersection with inherited_tags), regardless of whether the caller also happens to submit the identical value -- this is what set membership alone could never tell apart under the old subtraction-based approach. caller_tags is kept (documented as the complementary record) but no longer consulted for this decision. Verified live: a virtual key with metadata.tags=["®ion:eu"] hit with header x-litellm-tags: ®ion:eu,!region:eu (the exact value-collision attack) reproducibly routed to the OpenAI/us deployment before this fix and stayed on the Anthropic/eu deployment after it. * fix(lint): re-ratchet budget ceilings after rebasing onto litellm_internal_staging Regenerated ruff-strict-budget.json and type-discipline-budget.json via make lint-ruff-budget-update / lint-type-discipline-budget-update against the post-rebase merge-base. * fix(proxy): compute inherited_tags from key/team/project sources directly, not a tags-list snapshot apply_client_tag_policy_pre_auth (run from user_api_key_auth, for _tag_max_budget_check) merges the caller's x-litellm-tags header into the same metadata.tags list before add_litellm_data_to_request ever runs. The previous inherited_tags snapshot ("whatever's in tags before this function's own caller-tag merge") therefore misattributed that caller-controlled value as policy-backed whenever a request arrived with the header set -- Greptile flagged this as a P1 security finding. inherited_tags is now built directly from key_metadata/team_metadata/ project_metadata's own "tags" fields, independent of the shared, pipeline-position-dependent "tags" list's mutation history. Verified with a direct reproduction mirroring the real pipeline (calling apply_client_tag_policy_pre_auth on the same data dict before add_litellm_data_to_request, as user_api_key_auth actually does): the caller's header tag no longer appears in inherited_tags. Added a regression test exercising that same call order; confirmed it fails against the pre-fix snapshot approach and passes against this fix. * fix(router): make tag_routing_prefix configurable through update_settings/get_settings and UpdateRouterConfig router_settings.tag_routing_prefix was only ever applied via the Router() constructor. Router.update_settings's _allowed_settings (used directly by proxy_server.py's _add_router_settings_from_db_config for the DB-backed router_settings path) and get_settings's vars_to_include both omitted it, so an operator relying on that path had the value silently ignored -- flagged by veria-ai. Also adds it to UpdateRouterConfig (the pydantic schema behind POST /config/update), the same bug shape LIT-3152 previously fixed for retry_policy: a field missing from that schema gets silently dropped by model_dump(exclude_none=True) before update_settings is ever called. * chore(ui): regenerate schema.d.ts for UpdateRouterConfig.tag_routing_prefix Adding tag_routing_prefix to UpdateRouterConfig changed the proxy's OpenAPI spec; regenerate the dashboard's generated API types to match. * fix(lint): re-ratchet budget ceilings after rebasing onto litellm_internal_staging Regenerated ruff-strict-budget.json and type-discipline-budget.json against the post-rebase merge-base. LIT002/LIT011 ceilings reflect this branch's true current counts (confirmed unchanged across the rebase by diffing against the pre-rebase commit); the base's own counts moved independently. * fix(lint): replace mutable-collection fallbacks with immutable ones in inherited_tags computation key_metadata/team_metadata/project_metadata's "tags" fallbacks used `or {}` / `or []` literals, each a LIT002 mutable-collection-construction violation that pushed the branch 4 over its ratchet ceiling relative to a moved base. Swapped to MappingProxyType({}) / () to match the immutable idiom the rest of tag_based_routing.py already uses; no behavior change, since both are falsy and only ever read via .get()/ unpacking. Tightens type-discipline-budget.json's LIT002 ceiling back down to match, fully closing that gap (LIT011 keeps a genuine 1-count gap from pre-existing, untouched lines in this file, non-gating). * fix(lint): suppress LIT011 on the two new data[...] mutation sites Both new lines follow this file's established data[...] mutation idiom for add_litellm_data_to_request, matching the existing suppression already on the inherited_tags line. * test(router): lock in fallback + tag-filtering interaction Cover the router-level fallbacks mechanism composing with tag-based routing: a plain negation exhausting a group correctly advances to the fallback group, the same exclusion tag exhausting every hop correctly raises, and allow_fail_open resolving locally must not spuriously trigger an unrelated external fallback. * chore: retrigger CI now that litellm-docs#814 is merged --------- Co-authored-by: Deepanshu --- litellm/proxy/litellm_pre_call_utils.py | 49 +- litellm/router.py | 4 + litellm/router_strategy/tag_based_routing.py | 376 +++- litellm/types/router.py | 15 + ruff-strict-budget.json | 2 +- .../proxy/test_litellm_pre_call_utils.py | 236 +++ .../test_router_tag_regex_routing.py | 1 + .../test_router_tag_routing.py | 1717 ++++++++++++++++- type-discipline-budget.json | 6 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 + 10 files changed, 2346 insertions(+), 66 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 10142a894a1..0924b6aebea 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1862,6 +1862,24 @@ async def add_litellm_data_to_request( tags_to_add=project_metadata["tags"], ) + # inherited_tags: every tag key/team/project policy contributed, read + # directly from those three sources rather than snapshotted off the shared + # "tags" list. A pre-auth pass (apply_client_tag_policy_pre_auth, run from + # user_api_key_auth for _tag_max_budget_check) may already have merged the + # caller's own header tags into that same list before this function ever + # runs, so a snapshot taken here -- at any point in this function -- would + # misattribute caller-supplied tags as policy-backed. tag_based_routing.py's + # allow_fail_open reads this (rather than subtracting caller_tags from the + # final merged set) so a caller can't strip an inherited "!"/"&" + # constraint's protection just by resubmitting its exact value alongside a + # conflicting one. + _key_tags: Final = (key_metadata or MappingProxyType({})).get("tags") or () + _team_tags: Final = team_metadata.get("tags") or () + _project_tags: Final = project_metadata.get("tags") or () + data[_metadata_variable_name]["inherited_tags"] = tuple( # rebind-ok: matches this file's data[...] mutation idiom + dict.fromkeys((*_key_tags, *_team_tags, *_project_tags)) + ) + ## TEAM-LEVEL METADATA data = LiteLLMProxyRequestSetup.add_management_endpoint_metadata_to_request_metadata( data=data, @@ -1958,15 +1976,28 @@ async def add_litellm_data_to_request( tags_to_add=tags, ) - if _metadata_variable_name != "metadata": - _user_metadata = data.get("metadata") - if isinstance(_user_metadata, dict): - _user_tags: Final = _user_metadata.get("tags") - if isinstance(_user_tags, list) and _user_tags: - data[_metadata_variable_name]["tags"] = LiteLLMProxyRequestSetup._merge_tags( - request_tags=data[_metadata_variable_name].get("tags"), - tags_to_add=_user_tags, - ) + _caller_body_metadata: Final = data.get("metadata") if _metadata_variable_name != "metadata" else None + _caller_body_tags: Final = ( + _caller_body_metadata.get("tags") + if isinstance(_caller_body_metadata, dict) and isinstance(_caller_body_metadata.get("tags"), list) + else None + ) + if _caller_body_tags: + data[_metadata_variable_name]["tags"] = LiteLLMProxyRequestSetup._merge_tags( # rebind-ok: matches file idiom + request_tags=data[_metadata_variable_name].get("tags"), + tags_to_add=_caller_body_tags, + ) + + # caller_tags: exactly what this request itself supplied (x-litellm-tags header, + # body "tags", or body "metadata.tags" on litellm_metadata routes), never + # anything from key/team/project metadata. Read directly from the header and + # body values here, the same way inherited_tags above is read directly from + # key/team/project metadata -- neither is derived by inspecting the shared + # "tags" list, which a pre-auth pass (apply_client_tag_policy_pre_auth) may + # have already merged caller header tags into before this function runs. + data[_metadata_variable_name]["caller_tags"] = tuple( # rebind-ok: matches file idiom + dict.fromkeys((*(tags or ()), *(_caller_body_tags or ()))) + ) # Team Callbacks controls callback_settings_obj: Final = _get_dynamic_logging_metadata( diff --git a/litellm/router.py b/litellm/router.py index 9cece2014af..6af1c1a78e7 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -402,6 +402,7 @@ class Router: enable_pre_call_checks: bool = False, enable_tag_filtering: bool = False, tag_filtering_match_any: bool = True, + tag_routing_prefix: str = "", plugins: list[RoutingPlugin] | None = None, retry_after: int = 0, # min time to wait before retrying a failed request retry_policy: RetryPolicy | dict | None = None, # set custom retries for different exceptions @@ -511,6 +512,7 @@ class Router: self.enable_pre_call_checks = enable_pre_call_checks self.enable_tag_filtering = enable_tag_filtering self.tag_filtering_match_any = tag_filtering_match_any + self.tag_routing_prefix = tag_routing_prefix from litellm._service_logger import ServiceLogging self.service_logger_obj: ServiceLogging = ServiceLogging() @@ -10109,6 +10111,7 @@ class Router: "model_group_alias", "enable_weighted_failover", "enable_tag_filtering", + "tag_routing_prefix", ] for var in vars_to_include: @@ -10146,6 +10149,7 @@ class Router: "model_group_alias", "enable_weighted_failover", "enable_tag_filtering", + "tag_routing_prefix", ] _int_settings: Final = [ diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index c952b54e672..bbe97613c57 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -4,9 +4,12 @@ Use this to route requests between Teams - If tags in request is a subset of tags in deployment, return deployment - if deployments are set with default tags, return all default deployment - If no default_deployments are set, return all deployments +- A "!tag" excludes deployments carrying that tag; a "&tag" requires it """ import re +from collections.abc import Mapping, Sequence +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal from litellm._logging import verbose_logger @@ -114,14 +117,52 @@ def _match_deployment( return None -def _split_tags(tags: list[str]) -> tuple[list[str], list[str]]: - positive: Final = [t for t in tags if not t.startswith("!")] - excluded: Final = [tag[1:] for tag in tags if tag.startswith("!") and len(tag) > 1] - return positive, excluded +def _bare_tag_value(tag: str) -> str | None: + # Mirrors _split_tags' own stripping rule exactly, so a confirmed value + # compares equal to whatever required_set/excluded_set/positive_tags end up + # holding for the same tag: a "&"/"!" marker is stripped only when something + # follows it; a lone marker with nothing after it parses to nothing in any + # of the three sets, so it must not become a confirmed value either. + if tag.startswith(("&", "!")): + return tag[1:] if len(tag) > 1 else None + return tag + + +def _strip_routing_prefix(tags: Sequence[str], prefix: str) -> tuple[tuple[str, ...], frozenset[str]]: + # Strips the configured routing-prefix marker from any tag carrying it, used + # exactly as configured with no delimiter auto-appended, and separately + # tracks the post-strip, post-marker-strip values that arrived prefixed: tags + # whose routing intent the caller declared explicitly, exempt from the "maybe + # foreign to this group" heuristics in _unknown_required_tag_hides_an_answer + # and _tag_known_to_group below. Confirmed values are compared against + # required_set/excluded_set downstream, which are themselves already stripped + # of their "&"/"!" marker by _split_tags -- confirmed must match that same + # bare form, not the raw post-prefix-strip value that still carries the + # marker character. An empty prefix must return every tag unconfirmed, not + # run every tag through str.startswith(""), which is trivially True for + # every string and would mark everything confirmed. + if not prefix: + return tuple(tags), frozenset() + rewritten: Final = tuple(t.removeprefix(prefix) for t in tags) + confirmed: Final = frozenset( + bare + for bare in (_bare_tag_value(t.removeprefix(prefix)) for t in tags if t.startswith(prefix)) + if bare is not None + ) + return rewritten, confirmed + + +def _split_tags(tags: Sequence[str]) -> tuple[tuple[str, ...], list[str], tuple[str, ...]]: + required: Final = tuple(tag[1:] for tag in tags if tag.startswith("&") and len(tag) > 1) + positive: Final = [ + t for t in tags if not t.startswith("!") and not t.startswith("&") + ] # mutable-ok: feeds _match_deployment's existing list[str]-typed request_tags param + excluded: Final = tuple(tag[1:] for tag in tags if tag.startswith("!") and len(tag) > 1) + return required, positive, excluded def _exclude_deployments( - deployments: list[Any] | dict[Any, Any], + deployments: Sequence[Any] | Mapping[Any, Any], excluded_set: frozenset[str], ) -> list[Any]: if not excluded_set: @@ -129,24 +170,223 @@ def _exclude_deployments( return [d for d in deployments if not excluded_set.intersection(d.get("litellm_params", {}).get("tags") or [])] -def _require_candidates( - candidates: list[Any], +def _require_all_tags( + deployments: Sequence[Any] | Mapping[Any, Any], + required_set: frozenset[str], +) -> tuple[Any, ...]: + if not required_set: + return tuple(deployments) + return tuple(d for d in deployments if required_set.issubset(d.get("litellm_params", {}).get("tags") or [])) + + +def _default_tagged_pool( + deployments: Sequence[Any] | Mapping[Any, Any], +) -> tuple[Any, ...]: + defaults: Final = tuple(d for d in deployments if "default" in (d.get("litellm_params", {}).get("tags") or [])) + return defaults if defaults else tuple(deployments) + + +def _known_tag_values(deployments: Sequence[Any] | Mapping[Any, Any]) -> frozenset[str]: + return frozenset( + tag for d in deployments for tag in (d.get("litellm_params", MappingProxyType({})).get("tags") or ()) + ) + + +def _unknown_required_tag_hides_an_answer( + healthy_deployments: Sequence[Any] | Mapping[Any, Any], + excluded_set: frozenset[str], + required_set: frozenset[str], + routing_confirmed: frozenset[str], +) -> bool: + # A caller-invented "&" tag (one no deployment in this group has ever carried) + # guarantees an empty required-AND result on its own, regardless of whether the + # rest of the request's required tags were satisfiable. Dropping the unknown + # tags and recomputing: if that reveals a specific, non-empty answer, the invented + # tag was the actual cause of the exhaustion, and fail-open must not paper over + # it. If every required tag is known, or none are, there's nothing hidden to + # protect: either the caller made a real, honestly-unsatisfiable ask (fail-open + # proceeds normally), or the whole required set is unrecognized noise with no + # narrower answer to hide behind it. routing_confirmed (tag_routing_prefix) + # counts as known too: the caller explicitly declared it a routing directive, + # so it is never treated as invented noise regardless of deployment vocabulary. + known_required: Final = required_set & (_known_tag_values(healthy_deployments) | routing_confirmed) + if not known_required or known_required == required_set: + return False + allowed: Final = _exclude_deployments(healthy_deployments, excluded_set) + return bool(_require_all_tags(allowed, known_required)) + + +def _chain_allows_fail_open( + healthy_deployments: Sequence[Any] | Mapping[Any, Any], + excluded_set: frozenset[str], + required_set: frozenset[str], + routing_confirmed: frozenset[str], +) -> bool: + if _unknown_required_tag_hides_an_answer(healthy_deployments, excluded_set, required_set, routing_confirmed): + return False + return any((d.get("model_info") or {}).get("allow_fail_open") is True for d in healthy_deployments) + + +def _trusted_only_pool( + healthy_deployments: Sequence[Any] | Mapping[Any, Any], + excluded_set: frozenset[str], + required_set: frozenset[str], + inherited_excluded_set: frozenset[str] | None, + inherited_required_set: frozenset[str] | None, +) -> tuple[Any, ...]: + # inherited_*_set is None only when this request carries no origin information + # at all (e.g. direct SDK Router usage, bypassing the proxy layer that + # populates metadata.inherited_tags) -- treat every constraint as + # caller-controlled in that case (protected == empty), reproducing this + # function's pre-provenance behavior exactly: an unconditional fall-open to the + # full default-tagged pool, constraints discarded entirely. Otherwise, a tag + # value is protected the moment it has ANY inherited backing, even when the + # caller also happens to submit the identical value themselves -- set + # membership can't distinguish "this value came from policy" from "this value + # coincidentally matches policy," so presence in the inherited set (not + # absence from a caller-supplied set) is what must gate discardability. This + # is deliberately intersection with inherited_*_set, not subtraction of a + # caller-supplied set: subtraction would let a caller strip an inherited + # requirement's protection just by resubmitting its exact value alongside a + # conflicting one (e.g. inherited "®ion:eu" plus caller "®ion:eu" + # and "!region:eu" would otherwise cancel the inherited requirement out). + trusted_excluded: Final = ( + frozenset[str]() if inherited_excluded_set is None else inherited_excluded_set & excluded_set + ) + trusted_required: Final = ( + frozenset[str]() if inherited_required_set is None else inherited_required_set & required_set + ) + return _require_all_tags(_exclude_deployments(healthy_deployments, trusted_excluded), trusted_required) + + +def _resolve_or_fail_open( + pool: Sequence[Any], + healthy_deployments: Sequence[Any] | Mapping[Any, Any], + excluded_set: frozenset[str], + required_set: frozenset[str], + inherited_excluded_set: frozenset[str] | None, + inherited_required_set: frozenset[str] | None, + routing_confirmed: frozenset[str], model: str, - request_tags: Any, -) -> list[Any]: - if not candidates: - raise ValueError( - f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model={model} and tags={request_tags}" + request_tags: object, +) -> tuple[Any, ...]: + if pool: + return tuple(pool) + if _chain_allows_fail_open(healthy_deployments, excluded_set, required_set, routing_confirmed): + # Fall open only within whatever still satisfies whichever constraints + # trace back to key/team policy. A constraint with no inherited backing at + # all (or, when inherited_tags is unavailable, any constraint at all) can + # be discarded; one inherited from key/team policy cannot -- if that alone + # is unsatisfiable, raise instead of silently routing around it. + trusted_pool: Final = _trusted_only_pool( + healthy_deployments, excluded_set, required_set, inherited_excluded_set, inherited_required_set ) - return candidates + if trusted_pool: + return _default_tagged_pool(trusted_pool) + raise ValueError( + f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model={model} and tags={request_tags}" + ) -def _ban_only_base_pool( - deployments: list[Any] | dict[Any, Any], -) -> list[Any]: - # Mirrors untagged-request semantics so callers can't use !tags to escape the default pool. - defaults: Final = [d for d in deployments if "default" in (d.get("litellm_params", {}).get("tags") or [])] - return defaults if defaults else list(deployments) +def _resolve_constraint_only_pool( + healthy_deployments: Sequence[Any] | Mapping[Any, Any], + excluded_set: frozenset[str], + required_set: frozenset[str], + inherited_excluded_set: frozenset[str] | None, + inherited_required_set: frozenset[str] | None, + routing_confirmed: frozenset[str], + model: str, + request_tags: object, +) -> tuple[Any, ...]: + pool: Final = ( + _require_all_tags(_exclude_deployments(healthy_deployments, excluded_set), required_set) + if required_set + else _exclude_deployments(_default_tagged_pool(healthy_deployments), excluded_set) + ) + return _resolve_or_fail_open( + pool, + healthy_deployments, + excluded_set, + required_set, + inherited_excluded_set, + inherited_required_set, + routing_confirmed, + model, + request_tags, + ) + + +def _all_deployments_or_fallback( + llm_router_instance: LitellmRouter, + model: str, + fallback: Sequence[Any] | Mapping[Any, Any], +) -> Sequence[Any] | Mapping[Any, Any]: + try: + return llm_router_instance._get_all_deployments(model_name=model) + except Exception: # noqa: BLE001 # fail safe toward today's healthy-only behavior on lookup errors + return fallback + + +def _chain_tag_filtering_override( + llm_router_instance: LitellmRouter, + model: str, + healthy_deployments: Sequence[Any] | Mapping[Any, Any], +) -> bool | None: + # Resolved from every deployment configured for this model group, not just the + # ones that survived cooldown/health filtering (async_get_healthy_deployments + # filters cooldowns before calling get_deployments_for_tag) -- otherwise the + # sole deployment carrying this group's only explicit override loses its effect + # the moment it's transiently unhealthy, silently falling back to the + # router-wide default and letting an attacker disable a chain's tag policy by + # repeatedly failing that one deployment into cooldown. Falls back to + # healthy_deployments on a lookup error, preserving today's behavior rather + # than crashing the request. + all_deployments: Final = _all_deployments_or_fallback(llm_router_instance, model, healthy_deployments) + for d in all_deployments: + value = (d.get("model_info") or MappingProxyType({})).get("enable_tag_filtering") + if value is not None: + return value + return None + + +def _inherited_constraint_sets( + inherited_tags: object, routing_prefix: str +) -> tuple[frozenset[str] | None, frozenset[str] | None]: + # None means no origin information is available at all (e.g. this request + # bypassed the proxy layer that populates metadata.inherited_tags, as direct + # SDK Router usage does) -- callers of this must treat that as "nothing is + # protected," not "nothing is inherited," see _trusted_only_pool. + # metadata.inherited_tags is a snapshot of whatever key/team/project policy + # merged into "tags" *before* this request's own caller-supplied tags were + # merged in on top (see litellm_pre_call_utils.py), so a value present here is + # policy-backed regardless of whether the caller also happens to submit the + # identical value. inherited_tags is stripped through the same routing_prefix + # as the main request tags so a policy-inherited prefixed tag still matches + # correctly against the (already-stripped) required_set/excluded_set computed + # from request_tags. + if not isinstance(inherited_tags, (list, tuple)): + return None, None + rewritten_inherited_tags: Final = _strip_routing_prefix(inherited_tags, routing_prefix)[0] + inherited_required, _inherited_positive, inherited_excluded = _split_tags(rewritten_inherited_tags) + return frozenset(inherited_required), frozenset(inherited_excluded) + + +def _tag_known_to_group( + llm_router_instance: LitellmRouter, + model: str, + positive_tags: Sequence[str], + routing_confirmed: frozenset[str], +) -> bool: + tag_set: Final = frozenset(positive_tags) + if tag_set & routing_confirmed: + return True + try: + all_deployments: Final = llm_router_instance._get_all_deployments(model_name=model) + except Exception: # noqa: BLE001 # fail safe toward "unrecognized" so lookup errors preserve the existing silent-fallback behavior + return False + return any( + tag_set.intersection(d.get("litellm_params", MappingProxyType({})).get("tags") or ()) for d in all_deployments + ) async def get_deployments_for_tag( @@ -161,24 +401,29 @@ async def get_deployments_for_tag( Executes tag based filtering based on the tags in request metadata and the tags on the deployments - Runs when the router-level `enable_tag_filtering` is True or the request carries - `enable_tag_filtering=True` (set from key/team router_settings by the proxy). - A request-level False never disables a router-level True, so per-request settings - cannot escape an operator's global tag-routing policy. + Runs when the effective enable_tag_filtering is True. Effective value: a + request-level enable_tag_filtering=True (set from key/team router_settings by + the proxy) always wins; otherwise model_info.enable_tag_filtering on this model + group, if set on any of its deployments, overrides the router-wide default. + A request-level False never disables either of those, so per-request settings + cannot escape an operator's or a chain owner's tag-routing policy. """ - request_enable_tag_filtering: Final = request_kwargs.get("enable_tag_filtering") if request_kwargs else None - if request_enable_tag_filtering is not True and llm_router_instance.enable_tag_filtering is not True: - return healthy_deployments - - if request_kwargs is None: + if request_kwargs is None or not healthy_deployments: verbose_logger.debug( - "get_deployments_for_tag: request_kwargs is None returning healthy_deployments: %s", + "get_deployments_for_tag: skipping tag filter (request_kwargs=%s, healthy_deployments=%s)", + request_kwargs, healthy_deployments, ) return healthy_deployments - if not healthy_deployments: - verbose_logger.debug("get_deployments_for_tag: empty or None healthy_deployments; skipping tag filter") + request_enable_tag_filtering: Final = request_kwargs.get("enable_tag_filtering") + chain_enable_tag_filtering: Final = _chain_tag_filtering_override(llm_router_instance, model, healthy_deployments) + chain_default: Final = ( + chain_enable_tag_filtering + if chain_enable_tag_filtering is not None + else llm_router_instance.enable_tag_filtering + ) + if request_enable_tag_filtering is not True and chain_default is not True: return healthy_deployments verbose_logger.debug("request metadata: %s", request_kwargs.get(metadata_variable_name)) @@ -186,29 +431,52 @@ async def get_deployments_for_tag( metadata: Final = request_kwargs[metadata_variable_name] request_tags: Final = metadata.get("tags") match_any: Final = llm_router_instance.tag_filtering_match_any + routing_prefix: Final = llm_router_instance.tag_routing_prefix or "" # Build header strings for regex matching from what the proxy already stores. # Currently we match against User-Agent; format matches "^User-Agent: claude-code/..." user_agent: Final = metadata.get("user_agent", "") header_strings: Final[list[str]] = [f"User-Agent: {user_agent}"] if user_agent else [] - positive_tags, excluded_patterns = _split_tags(request_tags or []) + # A tag_routing_prefix-marked tag is stripped before matching -- everything + # downstream (_split_tags, deployment matching) works off the unprefixed + # value, exactly as if the caller had sent it unprefixed -- and its + # post-strip value is remembered in routing_confirmed as an explicit, + # caller-declared routing directive, exempt from the "maybe foreign to this + # group" heuristics that unprefixed tags still go through unchanged below. + rewritten_tags, routing_confirmed = _strip_routing_prefix(request_tags or [], routing_prefix) + required_tags, positive_tags, excluded_patterns = _split_tags(rewritten_tags) + inherited_required_set, inherited_excluded_set = _inherited_constraint_sets( + metadata.get("inherited_tags"), routing_prefix + ) excluded_set: Final = frozenset(excluded_patterns) - candidates: Final = _exclude_deployments(healthy_deployments, excluded_set) + required_set: Final = frozenset(required_tags) + allowed_deployments: Final = _exclude_deployments(healthy_deployments, excluded_set) + candidates: Final = _require_all_tags(allowed_deployments, required_set) has_regex_deployments: Final = any(d.get("litellm_params", {}).get("tag_regex") for d in candidates) - has_tag_filter: Final = bool(positive_tags) or (bool(header_strings) and has_regex_deployments) - ban_only: Final = bool(excluded_set) and not has_tag_filter + has_positive_filter: Final = bool(positive_tags) or ( + bool(header_strings) and has_regex_deployments and not required_set + ) + constraint_only: Final = (bool(excluded_set) or bool(required_set)) and not has_positive_filter - if ban_only: - pool: Final = _exclude_deployments(_ban_only_base_pool(healthy_deployments), excluded_set) - return _require_candidates(pool, model, request_tags) + if constraint_only: + return _resolve_constraint_only_pool( + healthy_deployments, + excluded_set, + required_set, + inherited_excluded_set, + inherited_required_set, + routing_confirmed, + model, + request_tags, + ) new_healthy_deployments: Final[list[Any]] = [] default_deployments: Final[list[Any]] = [] - if has_tag_filter: + if has_positive_filter: verbose_logger.debug( "get_deployments_for_tag routing: request_tags=%s user_agent=%s", request_tags, @@ -245,9 +513,33 @@ async def get_deployments_for_tag( default_deployments.append(deployment) if len(new_healthy_deployments) == 0 and len(default_deployments) == 0: - raise ValueError( - f"{RouterErrors.no_deployments_with_tag_routing.value}." - f" Passed model={model} and tags={request_tags}" + return _resolve_or_fail_open( + (), + healthy_deployments, + excluded_set, + required_set, + inherited_excluded_set, + inherited_required_set, + routing_confirmed, + model, + request_tags, + ) + + if ( + len(new_healthy_deployments) == 0 + and positive_tags + and _tag_known_to_group(llm_router_instance, model, positive_tags, routing_confirmed) + ): + return _resolve_or_fail_open( + (), + healthy_deployments, + excluded_set, + required_set, + inherited_excluded_set, + inherited_required_set, + routing_confirmed, + model, + request_tags, ) return new_healthy_deployments if len(new_healthy_deployments) > 0 else default_deployments diff --git a/litellm/types/router.py b/litellm/types/router.py index 3ac59c0f581..d7ff8d12aa6 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -123,6 +123,7 @@ class UpdateRouterConfig(BaseModel): context_window_fallbacks: list[dict] | None = None model_group_alias: dict[str, str | dict] | None = {} enable_tag_filtering: bool | None = None + tag_routing_prefix: str | None = None model_config = ConfigDict(protected_namespaces=()) @@ -170,6 +171,20 @@ class ModelInfo(MirroredPricingParams): ptu_effective_from: datetime.datetime | None = None ptu_effective_to: datetime.datetime | None = None + # when tag-based routing's "!" or "&" constraints eliminate every deployment + # in this model group, fall back to the default-tagged pool instead of + # raising no_deployments_with_tag_routing. Defaults to False (raise), so + # existing "!" negation behavior is unchanged unless explicitly opted in. + allow_fail_open: bool | None = None + + # per-model-group override for router_settings.enable_tag_filtering; unset + # defers to the router-wide default. Checked against any deployment in the + # group, so set it consistently across every deployment sharing this + # model_name. A request-level enable_tag_filtering=True (from key/team + # settings) still wins over this, exactly as it already does over the + # router-wide default. + enable_tag_filtering: bool | None = None + def __init__(self, id: str | int | None = None, **params) -> None: if id is None: id = str(uuid.uuid4()) # Generate a UUID if id is None or not provided diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index da0f608fdb5..12a98dee7ed 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -24,7 +24,7 @@ "limit": 133 }, "ANN401": { - "limit": 1495 + "limit": 1491 }, "ASYNC230": { "limit": 11 diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index f48e1dba601..293cce5fa5d 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -6309,3 +6309,239 @@ class TestPromotedTraceControlFields: assert "litellm_metadata" not in updated assert updated["metadata"]["trace_id"] == "trace-1" assert updated["metadata"]["session_id"] == "session-1" + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_inherited_tags_excludes_caller_tags(): + """inherited_tags must carry only what key/team/project policy contributed, + never anything the caller's own request (header/body) supplied, even when the + caller resubmits the identical value -- it's a snapshot taken before the + caller's own tags are merged in, not a set difference against caller_tags. + tag_based_routing.py's allow_fail_open relies on this so a caller can't strip + an inherited constraint's protection by resubmitting its exact value.""" + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/v1/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = { + "model": "gpt-3.5-turbo", + # Caller resubmits the exact value the key policy also contributes. + "tags": ["key-supplied"], + } + + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + user_id="real-user", + metadata={"tags": ["key-supplied"]}, + team_metadata={"tags": ["team-supplied"]}, + spend=0.0, + max_budget=100.0, + model_max_budget={}, + team_spend=0.0, + team_max_budget=200.0, + ) + + updated = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert set(updated["metadata"]["tags"]) == {"key-supplied", "team-supplied"} + assert set(updated["metadata"]["inherited_tags"]) == {"key-supplied", "team-supplied"} + assert tuple(updated["metadata"]["caller_tags"]) == ("key-supplied",) + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_inherited_tags_survives_pre_auth_header_merge(): + """Regression: apply_client_tag_policy_pre_auth (run from user_api_key_auth, + for _tag_max_budget_check) merges the caller's x-litellm-tags header into the + same metadata.tags list this function later reads from -- before this + function ever runs. A snapshot-based inherited_tags would misattribute that + caller-controlled value as policy-backed; inherited_tags must instead be read + directly from key/team/project metadata, immune to whatever the pre-auth pass + already merged into "tags".""" + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/v1/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json", "x-litellm-tags": "caller-invented-tag"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data: dict = {"model": "gpt-3.5-turbo"} + + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + user_id="real-user", + metadata={"tags": ["key-supplied"]}, + team_metadata={}, + spend=0.0, + max_budget=100.0, + model_max_budget={}, + team_spend=0.0, + team_max_budget=200.0, + ) + + # Simulate the real request pipeline: the pre-auth merge runs first, on the + # same data dict, before add_litellm_data_to_request is ever called. + LiteLLMProxyRequestSetup.apply_client_tag_policy_pre_auth( + request=request_mock, + request_data=data, + user_api_key_dict=user_api_key_dict, + ) + assert data["metadata"]["tags"] == ["caller-invented-tag"] + + updated = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert set(updated["metadata"]["tags"]) == {"caller-invented-tag", "key-supplied"} + assert updated["metadata"]["inherited_tags"] == ("key-supplied",) + assert updated["metadata"]["caller_tags"] == ("caller-invented-tag",) + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_caller_tags_excludes_key_and_team_tags(): + """caller_tags must carry only what the caller itself sent (header + body + tags), never anything merged in from key/team metadata, even though the + merged "tags" field (used for matching) legitimately contains all three.""" + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/v1/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = { + "model": "gpt-3.5-turbo", + "tags": ["caller-supplied"], + } + + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + user_id="real-user", + metadata={"tags": ["key-supplied"]}, + team_metadata={"tags": ["team-supplied"]}, + spend=0.0, + max_budget=100.0, + model_max_budget={}, + team_spend=0.0, + team_max_budget=200.0, + ) + + updated = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert set(updated["metadata"]["tags"]) == {"caller-supplied", "key-supplied", "team-supplied"} + assert tuple(updated["metadata"]["caller_tags"]) == ("caller-supplied",) + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_caller_tags_includes_header_tags(): + """The x-litellm-tags header is as much a caller-controlled input as the + body's "tags" field; both must land in caller_tags.""" + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/v1/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json", "x-litellm-tags": "header-tag"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = {"model": "gpt-3.5-turbo"} + + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + user_id="real-user", + metadata={"tags": ["key-supplied"]}, + team_metadata={}, + spend=0.0, + max_budget=100.0, + model_max_budget={}, + team_spend=0.0, + team_max_budget=200.0, + ) + + updated = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert set(updated["metadata"]["tags"]) == {"header-tag", "key-supplied"} + assert tuple(updated["metadata"]["caller_tags"]) == ("header-tag",) + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_caller_tags_empty_when_caller_sends_nothing(): + """caller_tags must be present (an empty tuple), not absent, when the caller + supplied no tags of their own -- an empty-but-present value tells + tag_based_routing.py's allow_fail_open that any required/excluded tag on the + request is entirely inherited, not that no origin information is available. + """ + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/v1/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = {"model": "gpt-3.5-turbo"} + + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + user_id="real-user", + metadata={"tags": ["key-supplied"]}, + team_metadata={}, + spend=0.0, + max_budget=100.0, + model_max_budget={}, + team_spend=0.0, + team_max_budget=200.0, + ) + + updated = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated["metadata"]["tags"] == ["key-supplied"] + assert updated["metadata"]["caller_tags"] == () diff --git a/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py b/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py index dca2bd84f92..6591478a4e7 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py @@ -112,6 +112,7 @@ def _make_router_mock(enable_tag_filtering=True, match_any=True): mock = MagicMock() mock.enable_tag_filtering = enable_tag_filtering mock.tag_filtering_match_any = match_any + mock.tag_routing_prefix = "" return mock diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py index 98506aad594..9e19e981f80 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py @@ -429,42 +429,58 @@ def test_get_tags_from_request_kwargs_various_inputs(): def test_split_tags_positive_only(): from litellm.router_strategy.tag_based_routing import _split_tags - positive, excluded = _split_tags(["paid", "teamA"]) + required, positive, excluded = _split_tags(["paid", "teamA"]) + assert required == () assert positive == ["paid", "teamA"] - assert excluded == [] + assert excluded == () def test_split_tags_negation_only(): from litellm.router_strategy.tag_based_routing import _split_tags - positive, excluded = _split_tags(["!provider:anthropic"]) + required, positive, excluded = _split_tags(["!provider:anthropic"]) + assert required == () assert positive == [] - assert excluded == ["provider:anthropic"] + assert excluded == ("provider:anthropic",) + + +def test_split_tags_required_only(): + from litellm.router_strategy.tag_based_routing import _split_tags + + required, positive, excluded = _split_tags(["&reasoning_type:high", "&provider:anthropic"]) + assert required == ("reasoning_type:high", "provider:anthropic") + assert positive == [] + assert excluded == () def test_split_tags_mixed(): from litellm.router_strategy.tag_based_routing import _split_tags - positive, excluded = _split_tags(["paid", "!provider:anthropic", "!inference:cerebras"]) + required, positive, excluded = _split_tags( + ["paid", "!provider:anthropic", "!inference:cerebras", "&reasoning_type:high"] + ) + assert required == ("reasoning_type:high",) assert positive == ["paid"] assert len(excluded) == 2 -def test_split_tags_bare_bang_skipped(): +def test_split_tags_bare_bang_and_amp_skipped(): from litellm.router_strategy.tag_based_routing import _split_tags - # A bare "!" with nothing after it is not a valid negation tag; skip it - positive, excluded = _split_tags(["paid", "!"]) + # A bare "!" or "&" with nothing after it is not a valid tag; skip it + required, positive, excluded = _split_tags(["paid", "!", "&"]) + assert required == () assert positive == ["paid"] - assert excluded == [] + assert excluded == () def test_split_tags_empty(): from litellm.router_strategy.tag_based_routing import _split_tags - positive, excluded = _split_tags([]) + required, positive, excluded = _split_tags([]) + assert required == () assert positive == [] - assert excluded == [] + assert excluded == () # --- get_deployments_for_tag negation integration tests --- @@ -1115,3 +1131,1682 @@ async def test_request_level_enable_tag_filtering_false_cannot_disable_global(): mock_response="hi", ) assert response._hidden_params["model_id"] == "team-a-deployment" + + +# --- model_info.enable_tag_filtering per-chain override --- + + +class _FakeRouterForChainOverride: + def __init__(self, all_deployments): + self._all_deployments = all_deployments + + def _get_all_deployments(self, model_name): + return self._all_deployments + + +def test_chain_tag_filtering_override_reads_any_member(): + from litellm.router_strategy.tag_based_routing import _chain_tag_filtering_override + + deployments = [ + {"model_info": {}}, + {"model_info": {"enable_tag_filtering": False}}, + ] + router = _FakeRouterForChainOverride(deployments) + assert _chain_tag_filtering_override(router, "gpt-4", deployments) is False + + +def test_chain_tag_filtering_override_none_when_unset_anywhere(): + from litellm.router_strategy.tag_based_routing import _chain_tag_filtering_override + + deployments = [{"model_info": {}}, {}] + router = _FakeRouterForChainOverride(deployments) + assert _chain_tag_filtering_override(router, "gpt-4", deployments) is None + + +def test_chain_tag_filtering_override_survives_the_overriding_member_going_unhealthy(): + # Regression: the per-group override must be resolved from every deployment + # configured for the model, not just the ones that survived cooldown/health + # filtering. async_get_healthy_deployments filters cooldowns before calling + # into get_deployments_for_tag, so healthy_deployments alone can be missing + # the one deployment that carries the group's only explicit override. + from litellm.router_strategy.tag_based_routing import _chain_tag_filtering_override + + all_deployments = [ + {"model_info": {"enable_tag_filtering": True}}, + {"model_info": {}}, + ] + router = _FakeRouterForChainOverride(all_deployments) + # The overriding deployment (index 0) is cooled down and absent from + # healthy_deployments -- the override must still be found via the full-group + # lookup, not silently lost. + healthy_deployments = [all_deployments[1]] + assert _chain_tag_filtering_override(router, "gpt-4", healthy_deployments) is True + + +def test_chain_tag_filtering_override_falls_back_to_healthy_deployments_on_lookup_error(): + from litellm.router_strategy.tag_based_routing import _chain_tag_filtering_override + + class _BrokenRouter: + def _get_all_deployments(self, model_name): + raise RuntimeError("model group not found") + + healthy_deployments = [{"model_info": {"enable_tag_filtering": False}}] + assert _chain_tag_filtering_override(_BrokenRouter(), "gpt-4", healthy_deployments) is False + + +@pytest.mark.asyncio() +async def test_chain_enable_tag_filtering_true_overrides_router_level_false(): + # Router-wide tag filtering is off; this model group opts in on its own via + # model_info.enable_tag_filtering, so tags still apply to requests for it. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["teamA"], + }, + "model_info": {"id": "team-a-deployment", "enable_tag_filtering": True}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["teamB"], + }, + "model_info": {"id": "team-b-deployment", "enable_tag_filtering": True}, + }, + ], + enable_tag_filtering=False, + ) + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["teamA"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "team-a-deployment" + + +@pytest.mark.asyncio() +async def test_chain_enable_tag_filtering_false_overrides_router_level_true(): + # Router-wide tag filtering is on, but this model group opts itself out via + # model_info.enable_tag_filtering: tags are ignored for requests to this group, + # so an untagged-style request just gets ordinary load-balanced routing. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["teamA"], + }, + "model_info": {"id": "team-a-deployment", "enable_tag_filtering": False}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["teamB"], + }, + "model_info": {"id": "team-b-deployment", "enable_tag_filtering": False}, + }, + ], + enable_tag_filtering=True, + ) + + seen_ids = set() + for _ in range(10): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["teamA"]}, + mock_response="hi", + ) + seen_ids.add(response._hidden_params["model_id"]) + + assert seen_ids == {"team-a-deployment", "team-b-deployment"} + + +@pytest.mark.asyncio() +async def test_request_level_enable_tag_filtering_still_wins_over_chain_level_false(): + # A key/team's own request-level enable_tag_filtering=True must still win over + # a chain that opted itself out, exactly as it already wins over the router + # default: request-level escalation is the highest-precedence layer. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["teamA"], + }, + "model_info": {"id": "team-a-deployment", "enable_tag_filtering": False}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["teamB"], + }, + "model_info": {"id": "team-b-deployment", "enable_tag_filtering": False}, + }, + ], + enable_tag_filtering=False, + ) + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["teamA"]}, + enable_tag_filtering=True, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "team-a-deployment" + + +# --- _require_all_tags / _chain_allows_fail_open unit tests --- + + +def test_require_all_tags_empty_required_set_is_noop(): + from litellm.router_strategy.tag_based_routing import _require_all_tags + + deployments = [{"litellm_params": {"tags": ["a"]}}, {"litellm_params": {"tags": []}}] + assert _require_all_tags(deployments, frozenset()) == tuple(deployments) + + +def test_require_all_tags_keeps_only_deployments_with_every_required_tag(): + from litellm.router_strategy.tag_based_routing import _require_all_tags + + has_both = {"litellm_params": {"tags": ["reasoning_type:high", "provider:anthropic"]}} + has_one = {"litellm_params": {"tags": ["reasoning_type:high"]}} + has_neither = {"litellm_params": {"tags": ["provider:openai"]}} + + result = _require_all_tags( + [has_both, has_one, has_neither], frozenset({"reasoning_type:high", "provider:anthropic"}) + ) + assert result == (has_both,) + + +def test_chain_allows_fail_open_true_when_any_member_sets_flag(): + from litellm.router_strategy.tag_based_routing import _chain_allows_fail_open + + deployments = [ + {"model_info": {}, "litellm_params": {"tags": ["provider:anthropic"]}}, + {"model_info": {"allow_fail_open": True}, "litellm_params": {"tags": ["provider:openai"]}}, + ] + assert _chain_allows_fail_open(deployments, frozenset(), frozenset({"provider:anthropic"}), frozenset()) is True + + +def test_chain_allows_fail_open_false_by_default(): + from litellm.router_strategy.tag_based_routing import _chain_allows_fail_open + + deployments = [{"model_info": {}}, {}] + assert _chain_allows_fail_open(deployments, frozenset(), frozenset(), frozenset()) is False + + +def test_chain_allows_fail_open_true_when_no_required_tag_is_known_at_all(): + # An entirely-invented required tag with nothing else known to compare against + # has no narrower answer to hide; a single-deployment catch-all fallback is a + # legitimate use of allow_fail_open, not something to deny. + from litellm.router_strategy.tag_based_routing import _chain_allows_fail_open + + deployments = [ + {"model_info": {"allow_fail_open": True}, "litellm_params": {"tags": ["default", "reasoning_type:low"]}}, + ] + assert _chain_allows_fail_open(deployments, frozenset(), frozenset({"reasoning_type:high"}), frozenset()) is True + + +def test_unknown_required_tag_hides_an_answer_denies_fail_open(): + from litellm.router_strategy.tag_based_routing import _chain_allows_fail_open + + deployments = [ + { + "model_info": {}, + "litellm_params": {"tags": ["provider:anthropic", "region:us-east"]}, + }, + { + "model_info": {"allow_fail_open": True}, + "litellm_params": {"tags": ["default", "provider:openai"]}, + }, + ] + # region:us-east is real and satisfiable on the first deployment; the invented tag + # alone forces emptiness. Dropping it reveals a specific, non-default answer, so + # fail-open must be denied even though the flag is set on the group. + assert ( + _chain_allows_fail_open( + deployments, frozenset(), frozenset({"region:us-east", "totally-invented-tag-nobody-has"}), frozenset() + ) + is False + ) + + +def test_unknown_required_tag_allows_fail_open_when_no_answer_is_hidden(): + from litellm.router_strategy.tag_based_routing import _chain_allows_fail_open + + deployments = [ + { + "model_info": {"allow_fail_open": True}, + "litellm_params": {"tags": ["provider:anthropic", "region:us-east"]}, + }, + { + "model_info": {"allow_fail_open": True}, + "litellm_params": {"tags": ["provider:eu", "region:eu"]}, + }, + { + "model_info": {"allow_fail_open": True}, + "litellm_params": {"tags": ["default", "provider:openai"]}, + }, + ] + # region:us-east and region:eu are both real, known tags; no single deployment + # carries both, so this is a genuinely unsatisfiable combination, not an invented + # tag masking a narrower answer. Fail-open must proceed normally. + assert ( + _chain_allows_fail_open(deployments, frozenset(), frozenset({"region:us-east", "region:eu"}), frozenset()) + is True + ) + + +# --- _strip_routing_prefix / _bare_tag_value unit tests --- + + +def test_strip_routing_prefix_empty_prefix_is_noop(): + from litellm.router_strategy.tag_based_routing import _strip_routing_prefix + + tags = ["provider:anthropic", "®ion:eu", "!region:us"] + rewritten, confirmed = _strip_routing_prefix(tags, "") + assert rewritten == tuple(tags) + assert confirmed == frozenset() + + +def test_strip_routing_prefix_splits_routed_from_other(): + from litellm.router_strategy.tag_based_routing import _strip_routing_prefix + + rewritten, confirmed = _strip_routing_prefix(["feature:demo", "route:!provider:openai"], "route:") + assert rewritten == ("feature:demo", "!provider:openai") + assert confirmed == frozenset({"provider:openai"}) + + +def test_strip_routing_prefix_confirmed_matches_bare_required_and_excluded_values(): + # Regression: confirmed must carry the same bare (marker-stripped) form that + # _split_tags produces for required_set/excluded_set downstream. A prior bug + # left the "&"/"!" marker in `confirmed`, so `required_set & routing_confirmed` + # never intersected for any prefixed "&"/"!" tag -- the entire "trusted, + # caller-declared required/excluded tag" mechanism silently no-opped. + from litellm.router_strategy.tag_based_routing import _strip_routing_prefix + + _, confirmed = _strip_routing_prefix(["route:&provider:anthropic", "route:!region:eu"], "route:") + assert confirmed == frozenset({"provider:anthropic", "region:eu"}) + + +def test_strip_routing_prefix_lone_marker_confirms_nothing(): + from litellm.router_strategy.tag_based_routing import _strip_routing_prefix + + # A lone "&"/"!" with nothing after it parses to nothing in required_set, + # excluded_set, or positive_tags (see test_split_tags_bare_bang_and_amp_skipped); + # confirmed must not invent a value for it either. + _, confirmed = _strip_routing_prefix(["route:&", "route:!"], "route:") + assert confirmed == frozenset() + + +def test_chain_allows_fail_open_true_when_prefixed_unknown_required_tag_is_confirmed(): + # Regression for the same bug: a required tag no deployment carries is normally + # treated as invented noise that can hide a narrower answer (see + # test_unknown_required_tag_hides_an_answer_denies_fail_open) -- but once the + # caller has explicitly marked it via the routing prefix, it counts as a known, + # honest ask, and fail-open must proceed rather than get denied. + from litellm.router_strategy.tag_based_routing import _chain_allows_fail_open + + deployments = [ + { + "model_info": {"allow_fail_open": True}, + "litellm_params": {"tags": ["default", "provider:anthropic"]}, + }, + ] + required_set = frozenset({"provider:anthropic", "typo-tag"}) + assert _chain_allows_fail_open(deployments, frozenset(), required_set, frozenset()) is False + assert _chain_allows_fail_open(deployments, frozenset(), required_set, required_set) is True + + +# --- get_deployments_for_tag required-AND ("&") integration tests --- + + +@pytest.mark.asyncio() +async def test_required_and_matches_deployment_with_all_tags(): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:high", "provider:anthropic"], + }, + "model_info": {"id": "high-reasoning-anthropic"}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:high", "provider:openai"], + }, + "model_info": {"id": "high-reasoning-openai"}, + }, + ], + enable_tag_filtering=True, + ) + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["&reasoning_type:high", "&provider:anthropic"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "high-reasoning-anthropic" + + +@pytest.mark.asyncio() +async def test_required_and_excludes_deployment_missing_one_tag(): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:high", "provider:anthropic"], + }, + "model_info": {"id": "high-reasoning-anthropic"}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:low", "provider:anthropic"], + }, + "model_info": {"id": "low-reasoning-anthropic"}, + }, + ], + enable_tag_filtering=True, + ) + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["&reasoning_type:high", "&provider:anthropic"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "high-reasoning-anthropic" + + +@pytest.mark.asyncio() +async def test_required_and_composes_with_negation(): + # &reasoning_type:high requires the tag; !provider:anthropic bans that provider. + # Negation applies first, so the anthropic deployment is excluded even though + # it satisfies the required tag. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:high", "provider:anthropic"], + }, + "model_info": {"id": "high-reasoning-anthropic"}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:high", "provider:openai"], + }, + "model_info": {"id": "high-reasoning-openai"}, + }, + ], + enable_tag_filtering=True, + ) + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["&reasoning_type:high", "!provider:anthropic"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "high-reasoning-openai" + + +@pytest.mark.asyncio() +async def test_required_and_combines_with_positive_or_preference(): + # &reasoning_type:high is a hard requirement; provider:anthropic/provider:openai + # is a preference (OR) applied on top of the survivors. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:high", "provider:anthropic"], + }, + "model_info": {"id": "high-reasoning-anthropic"}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:high", "provider:vertex"], + }, + "model_info": {"id": "high-reasoning-vertex"}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:low", "provider:anthropic"], + }, + "model_info": {"id": "low-reasoning-anthropic"}, + }, + ], + enable_tag_filtering=True, + ) + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["&reasoning_type:high", "provider:anthropic", "provider:openai"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "high-reasoning-anthropic" + + +@pytest.mark.asyncio() +async def test_required_and_single_tag_matches_trivially(): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:high"], + }, + "model_info": {"id": "high-reasoning"}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:low"], + }, + "model_info": {"id": "low-reasoning"}, + }, + ], + enable_tag_filtering=True, + ) + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["&reasoning_type:high"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "high-reasoning" + + +@pytest.mark.asyncio() +async def test_required_and_unmatched_raises_by_default(): + # allow_fail_open unset -> unmatched required-AND raises, same as today's "!" behavior. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:low"], + }, + "model_info": {"id": "low-reasoning"}, + }, + ], + enable_tag_filtering=True, + ) + + with pytest.raises(Exception) as exc_info: + await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["&reasoning_type:high"]}, + mock_response="hi", + ) + + from litellm.types.router import RouterErrors + + assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value) + + +@pytest.mark.asyncio() +async def test_required_and_combined_with_positive_unmatched_raises_by_default(): + # &A eliminates every candidate before the positive-tag preference even runs; + # this must be gated by allow_fail_open too, not just the required-AND-only path. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:low", "provider:anthropic"], + }, + "model_info": {"id": "low-reasoning-anthropic"}, + }, + ], + enable_tag_filtering=True, + ) + + with pytest.raises(Exception) as exc_info: + await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["&reasoning_type:high", "provider:anthropic"]}, + mock_response="hi", + ) + + from litellm.types.router import RouterErrors + + assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value) + + +# --- get_deployments_for_tag allow_fail_open integration tests --- + + +@pytest.mark.asyncio() +async def test_allow_fail_open_required_and_unmatched_falls_back_to_default_pool(): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["default", "reasoning_type:low"], + }, + "model_info": {"id": "default-model", "allow_fail_open": True}, + }, + ], + enable_tag_filtering=True, + ) + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["&reasoning_type:high"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "default-model" + + +@pytest.mark.asyncio() +async def test_allow_fail_open_negation_eliminates_everything_includes_banned_deployment(): + # The core backwards-compatibility risk: once allow_fail_open opts a chain in, + # a "!" ban that eliminates every deployment falls back to the full default + # pool, INCLUDING the deployment the request tried to ban. This must never + # silently disappear (still raise) nor silently reappear on chains without + # the flag set (see test_negation_all_excluded_raises). + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["provider:anthropic"], + }, + "model_info": {"id": "anthropic-model", "allow_fail_open": True}, + }, + ], + enable_tag_filtering=True, + ) + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["!provider:anthropic"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "anthropic-model" + + +@pytest.mark.asyncio() +async def test_allow_fail_open_prefers_default_tagged_deployment_on_fallback(): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["provider:anthropic"], + }, + "model_info": {"id": "anthropic-model", "allow_fail_open": True}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["provider:anthropic", "default"], + }, + "model_info": {"id": "anthropic-default-model", "allow_fail_open": True}, + }, + ], + enable_tag_filtering=True, + ) + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["!provider:anthropic"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "anthropic-default-model" + + +@pytest.mark.asyncio() +async def test_allow_fail_open_per_hop_across_fallback_chain(): + # required-AND fail-open must be re-evaluated fresh on every hop, the same + # per-hop guarantee the negation feature already established. + router = litellm.Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:low"], + }, + "model_info": {"id": "primary-low-reasoning"}, + }, + { + "model_name": "fallback", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["default", "reasoning_type:low"], + }, + "model_info": {"id": "fallback-model", "allow_fail_open": True}, + }, + ], + fallbacks=[{"primary": ["fallback"]}], + enable_tag_filtering=True, + ) + + response = await router.acompletion( + model="primary", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["&reasoning_type:high"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "fallback-model" + + +@pytest.mark.asyncio() +async def test_allow_fail_open_resolves_locally_without_triggering_external_fallback(): + # allow_fail_open on the primary group's own default deployment absorbs the + # exhaustion internally (_resolve_or_fail_open returns a non-empty pool, so + # get_deployments_for_tag never raises); router.async_function_with_fallbacks + # only invokes the configured "fallbacks" chain on an exception, so a + # separate, unrelated fallback group must never be touched even though one is + # configured. A fallback deployment that would trivially satisfy the request + # tag if it were ever consulted makes this a meaningful negative assertion, + # not a vacuous one. + router = litellm.Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:high"], + }, + "model_info": {"id": "primary-high-reasoning"}, + }, + { + "model_name": "primary", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["default", "reasoning_type:low"], + }, + "model_info": {"id": "primary-default", "allow_fail_open": True}, + }, + { + "model_name": "fallback", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["region:eu"], + }, + "model_info": {"id": "fallback-should-never-be-used"}, + }, + ], + fallbacks=[{"primary": ["fallback"]}], + enable_tag_filtering=True, + ) + + response = await router.acompletion( + model="primary", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["®ion:eu"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "primary-default" + + +# --- allow_fail_open must also gate "!" exhaustion combined with a plain positive tag --- + + +@pytest.mark.asyncio() +async def test_negation_combined_with_positive_unmatched_raises_by_default(): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["provider:anthropic", "paid"], + }, + "model_info": {"id": "anthropic-paid"}, + }, + ], + enable_tag_filtering=True, + ) + + with pytest.raises(Exception) as exc_info: + await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["!provider:anthropic", "paid"]}, + mock_response="hi", + ) + + from litellm.types.router import RouterErrors + + assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value) + + +@pytest.mark.asyncio() +async def test_negation_combined_with_positive_unmatched_falls_open_when_allowed(): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["provider:anthropic", "paid", "default"], + }, + "model_info": {"id": "anthropic-paid", "allow_fail_open": True}, + }, + ], + enable_tag_filtering=True, + ) + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["!provider:anthropic", "paid"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "anthropic-paid" + + +# --- a required-AND-only request must not be diluted by incidental regex/header preference --- + + +@pytest.mark.asyncio() +async def test_required_and_only_returns_every_matching_deployment_despite_regex_header(): + # Deployment A satisfies &reasoning_type:high and also happens to carry a tag_regex + # that matches the caller's User-Agent. Deployment B also satisfies the required tag + # but has no tag_regex at all. A required-AND-only request (no plain positive tags) + # must be free to route to either survivor, not be narrowed down to only the one + # that happens to match the incidental regex/header preference. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:high"], + "tag_regex": ["^User-Agent: claude-code\\/"], + }, + "model_info": {"id": "high-reasoning-with-regex"}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:high"], + }, + "model_info": {"id": "high-reasoning-no-regex"}, + }, + ], + enable_tag_filtering=True, + ) + + seen_ids = set() + for _ in range(30): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["&reasoning_type:high"], "user_agent": "claude-code/1.2.3"}, + mock_response="hi", + ) + seen_ids.add(response._hidden_params["model_id"]) + + assert seen_ids == {"high-reasoning-with-regex", "high-reasoning-no-regex"} + + +@pytest.mark.asyncio() +async def test_required_and_only_excludes_regex_deployment_missing_the_required_tag(): + # The tag_regex deployment matches the caller's User-Agent but does NOT carry the + # required tag; a required-AND-only request must not let it through on the strength + # of the regex/header match alone. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:low"], + "tag_regex": ["^User-Agent: claude-code\\/"], + }, + "model_info": {"id": "low-reasoning-with-regex"}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:high"], + }, + "model_info": {"id": "high-reasoning-no-regex"}, + }, + ], + enable_tag_filtering=True, + ) + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["&reasoning_type:high"], "user_agent": "claude-code/1.2.3"}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "high-reasoning-no-regex" + + +# --- allow_fail_open must also gate exhaustion after a non-empty required-AND survivor +# set fails to match a plain preference tag, not just full !/& exhaustion --- + + +@pytest.mark.asyncio() +async def test_mixed_constraint_survivor_unmatched_by_positive_tag_raises_by_default(): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:high", "provider:anthropic"], + }, + "model_info": {"id": "high-reasoning-anthropic"}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["default", "reasoning_type:low"], + }, + "model_info": {"id": "default-fallback"}, + }, + ], + enable_tag_filtering=True, + ) + + with pytest.raises(Exception) as exc_info: + await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["&reasoning_type:high", "provider:openai"]}, + mock_response="hi", + ) + + from litellm.types.router import RouterErrors + + assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value) + + +@pytest.mark.asyncio() +async def test_mixed_constraint_survivor_unmatched_by_positive_tag_falls_open_when_allowed(): + # &reasoning_type:high survives to a non-empty candidate set (the anthropic + # deployment), but the plain preference tag provider:openai matches none of the + # survivors, and the surviving deployment itself is not "default"-tagged (so the + # pre-existing in-loop default-collection escape hatch can't mask the fix). Greptile + # flagged this exact path as bypassing allow_fail_open by raising unconditionally; + # it must instead fall back to the group's actual default-tagged deployment, which + # is a different deployment than the one &reasoning_type:high matched. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:high", "provider:anthropic"], + }, + "model_info": {"id": "high-reasoning-anthropic", "allow_fail_open": True}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["default", "reasoning_type:low"], + }, + "model_info": {"id": "default-fallback", "allow_fail_open": True}, + }, + ], + enable_tag_filtering=True, + ) + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["&reasoning_type:high", "provider:openai"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "default-fallback" + + +# --- allow_fail_open must not be triggerable by an invented tag the chain has never +# carried; a caller-supplied garbage tag must not be able to force an otherwise- +# satisfiable constraint (e.g. one inherited from the key/team) to be discarded --- + + +@pytest.mark.asyncio() +async def test_allow_fail_open_denied_when_request_includes_unknown_tag(): + # region:us-east is a real, satisfiable constraint on anthropic-deployment. Adding + # a single invented tag no deployment in this group has ever carried empties the + # required-AND set regardless of region:us-east's own satisfiability. allow_fail_open + # is set on the default deployment, but must not fire here: none of the *other* + # deployments carry the invented tag either, so it is unknown to the chain, and + # falling back would silently discard the still-satisfiable region:us-east ask. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["provider:anthropic", "region:us-east"], + }, + "model_info": {"id": "anthropic-deployment"}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["default", "provider:openai"], + }, + "model_info": {"id": "openai-default", "allow_fail_open": True}, + }, + ], + enable_tag_filtering=True, + ) + + with pytest.raises(Exception) as exc_info: + await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["®ion:us-east", "&totally-invented-tag-nobody-has"]}, + mock_response="hi", + ) + + from litellm.types.router import RouterErrors + + assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value) + + +@pytest.mark.asyncio() +async def test_allow_fail_open_still_fires_when_every_requested_tag_is_known(): + # region:us-east and region:eu are both real tags this chain uses; no single + # deployment carries both, so the combination is genuinely unsatisfiable, not + # invented. allow_fail_open must still fall back normally in this case. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["provider:anthropic", "region:us-east"], + }, + "model_info": {"id": "anthropic-deployment", "allow_fail_open": True}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["provider:eu", "region:eu"], + }, + "model_info": {"id": "eu-deployment", "allow_fail_open": True}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["default", "provider:openai"], + }, + "model_info": {"id": "openai-default", "allow_fail_open": True}, + }, + ], + enable_tag_filtering=True, + ) + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["®ion:us-east", "®ion:eu"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "openai-default" + + +# --- required-AND, allow_fail_open, and the unknown-tag denial across fallback +# chains spanning multiple model groups --- + + +@pytest.mark.asyncio() +async def test_required_and_exhausts_primary_group_falls_through_to_fallback_group(): + # &reasoning_type:high matches nothing on "primary" (raises internally, same as + # negation's own fallback-chain behavior), so the router advances to "fallback" + # where the tag is satisfiable. No allow_fail_open involved; this is the plain + # fallback-chain mechanics already established for "!" extended to "&". + router = litellm.Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:low"], + }, + "model_info": {"id": "primary-low-reasoning"}, + }, + { + "model_name": "fallback", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["reasoning_type:high"], + }, + "model_info": {"id": "fallback-high-reasoning"}, + }, + ], + fallbacks=[{"primary": ["fallback"]}], + enable_tag_filtering=True, + ) + + response = await router.acompletion( + model="primary", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["&reasoning_type:high"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "fallback-high-reasoning" + + +@pytest.mark.asyncio() +async def test_required_and_negation_and_allow_fail_open_combine_across_three_model_groups(): + # A single request routes through three independent model groups via two + # fallback hops, exercising "!", "&", and allow_fail_open together at each hop: + # - "primary" is banned outright by "!provider:anthropic" -> raises, advances. + # - "secondary" satisfies the negation but not &reasoning_type:high, and has no + # allow_fail_open -> raises exactly as today, advances. + # - "tertiary" has reasoning_type:high, but only on the deployment the same + # "!provider:anthropic" also bans; the tag is known to the chain but its only + # carrier is legitimately excluded, not hidden behind an invented tag, so the + # opted-in allow_fail_open falls back to the group's own default deployment. + router = litellm.Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["provider:anthropic", "reasoning_type:high"], + }, + "model_info": {"id": "primary-anthropic"}, + }, + { + "model_name": "secondary", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["provider:openai", "reasoning_type:low"], + }, + "model_info": {"id": "secondary-openai"}, + }, + { + "model_name": "tertiary", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["provider:anthropic", "reasoning_type:high", "region:eu"], + }, + "model_info": {"id": "tertiary-anthropic-high-reasoning"}, + }, + { + "model_name": "tertiary", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["default", "provider:openai", "reasoning_type:low"], + }, + "model_info": {"id": "tertiary-default", "allow_fail_open": True}, + }, + ], + fallbacks=[{"primary": ["secondary"]}, {"secondary": ["tertiary"]}], + enable_tag_filtering=True, + ) + + response = await router.acompletion( + model="primary", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["!provider:anthropic", "&reasoning_type:high"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "tertiary-default" + + +@pytest.mark.asyncio() +async def test_unknown_tag_denial_is_scoped_per_hop_not_leaked_across_fallback_groups(): + # On "primary": region:us-east is real and satisfiable there, but the invented + # tag masks it -> denies fail-open -> raises -> advances to "fallback". + # On "fallback": neither region:us-east nor the invented tag is known to this + # entirely different, unrelated group at all, so there's no answer for the + # invented tag to hide -> falls open normally. Each hop must independently + # discover what its own group knows; a deny decision from a prior hop's group + # must not leak forward and block a later hop that has no relevant knowledge. + router = litellm.Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["region:us-east"], + }, + "model_info": {"id": "primary-us-east", "allow_fail_open": True}, + }, + { + "model_name": "fallback", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["default", "provider:openai"], + }, + "model_info": {"id": "fallback-default", "allow_fail_open": True}, + }, + ], + fallbacks=[{"primary": ["fallback"]}], + enable_tag_filtering=True, + ) + + response = await router.acompletion( + model="primary", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["®ion:us-east", "&totally-invented-tag-nobody-has"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "fallback-default" + + +@pytest.mark.asyncio() +async def test_required_and_only_finds_compliant_non_default_deployment_over_noncompliant_default(): + # A required-AND-only request must be checked against every deployment in the + # group, not just the one tagged "default". A compliant, healthy deployment that + # simply isn't the operator's default must win over routing to a noncompliant + # default just because allow_fail_open happened to be set. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["provider:anthropic", "region:us-east"], + }, + "model_info": {"id": "anthropic-us-east"}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["default", "provider:openai"], + }, + "model_info": {"id": "openai-default", "allow_fail_open": True}, + }, + ], + enable_tag_filtering=True, + ) + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["®ion:us-east"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "anthropic-us-east" + + +# --- plain positive-tag exhaustion must not be masked by a universally-applied +# "default" tag; allow_fail_open must still be consulted (or hard-fail without it) --- + + +def _quality_high_cost_low_router(): + return litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["default", "quality:high"], + }, + "model_info": {"id": "quality-high-1"}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["default", "quality:high"], + }, + "model_info": {"id": "quality-high-2"}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["default", "cost:low"], + }, + "model_info": {"id": "cost-low-1"}, + }, + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["default", "cost:low"], + }, + "model_info": {"id": "cost-low-2"}, + }, + ], + enable_tag_filtering=True, + ) + + +@pytest.mark.asyncio() +async def test_plain_tag_exhaustion_with_universal_default_tag_raises_by_default(): + # Every deployment in the group is tagged "default" (a legitimate cross-cutting + # safety-net pattern), so default_deployments is never empty on its own. With + # the quality:high deployments unhealthy, a request asking for quality:high + # must still hard-fail, not silently get served by a cost:low deployment just + # because it happens to also carry "default". + from unittest.mock import AsyncMock, patch + + router = _quality_high_cost_low_router() + + with patch( + "litellm.router._async_get_cooldown_deployments", + new=AsyncMock(return_value=["quality-high-1", "quality-high-2"]), + ): + with pytest.raises(Exception) as exc_info: + await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["quality:high"]}, + mock_response="hi", + ) + + from litellm.types.router import RouterErrors + + assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value) + + +@pytest.mark.asyncio() +async def test_plain_tag_exhaustion_with_universal_default_tag_falls_open_when_allowed(): + router = _quality_high_cost_low_router() + for deployment in router.model_list: + deployment["model_info"]["allow_fail_open"] = True + + from unittest.mock import AsyncMock, patch + + with patch( + "litellm.router._async_get_cooldown_deployments", + new=AsyncMock(return_value=["quality-high-1", "quality-high-2"]), + ): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["quality:high"]}, + mock_response="hi", + ) + + assert response._hidden_params["model_id"] in ("cost-low-1", "cost-low-2") + + +@pytest.mark.asyncio() +async def test_plain_tag_unknown_to_group_still_falls_back_silently_unconditionally(): + # A tag that no deployment in this group has ever carried (foreign to this + # group entirely, e.g. an attribution tag meant for an unrelated mechanism + # sharing the same request-tags list) must keep falling back to the + # "default"-tagged pool unconditionally, exactly like today, regardless of + # allow_fail_open. Only a tag that IS part of this group's real vocabulary + # triggers the new hard-fail/fail-open gate. + router = _quality_high_cost_low_router() + + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["llm-preference-include:some-unrelated-mechanism"]}, + mock_response="hi", + ) + assert response._hidden_params["model_id"] in ( + "quality-high-1", + "quality-high-2", + "cost-low-1", + "cost-low-2", + ) + + +def test_tag_known_to_group_true_for_real_tag(): + from litellm.router_strategy.tag_based_routing import _tag_known_to_group + + router = _quality_high_cost_low_router() + assert _tag_known_to_group(router, "gpt-4", ["quality:high"], frozenset()) is True + + +def test_tag_known_to_group_false_for_foreign_tag(): + from litellm.router_strategy.tag_based_routing import _tag_known_to_group + + router = _quality_high_cost_low_router() + assert _tag_known_to_group(router, "gpt-4", ["llm-preference-include:unrelated"], frozenset()) is False + + +def test_inherited_constraint_sets_none_when_inherited_tags_absent(): + from litellm.router_strategy.tag_based_routing import _inherited_constraint_sets + + assert _inherited_constraint_sets(None, "") == (None, None) + + +def test_inherited_constraint_sets_splits_required_and_excluded(): + from litellm.router_strategy.tag_based_routing import _inherited_constraint_sets + + inherited_required_set, inherited_excluded_set = _inherited_constraint_sets( + ["®ion:eu", "!region:us", "plain"], "" + ) + assert inherited_required_set == frozenset({"region:eu"}) + assert inherited_excluded_set == frozenset({"region:us"}) + + +def test_inherited_constraint_sets_none_for_non_sequence_value(): + from litellm.router_strategy.tag_based_routing import _inherited_constraint_sets + + # A malformed/unexpected inherited_tags value (anything but a list/tuple) must + # be treated the same as "no origin information", never as "nothing is + # inherited" -- the two are not interchangeable, see _trusted_only_pool. + assert _inherited_constraint_sets("not-a-sequence", "") == (None, None) + + +def test_trusted_only_pool_discards_everything_when_inherited_sets_are_none(): + from litellm.router_strategy.tag_based_routing import _trusted_only_pool + + deployments = ({"litellm_params": {"tags": ["region:us"]}},) + # No origin info at all -> reproduce the pre-provenance unconditional + # fall-open: the trusted-only pool ignores excluded_set/required_set entirely. + assert _trusted_only_pool(deployments, frozenset({"region:eu"}), frozenset({"region:apac"}), None, None) == deployments + + +def test_trusted_only_pool_keeps_constraint_backed_by_inherited_tags(): + from litellm.router_strategy.tag_based_routing import _trusted_only_pool + + eu = {"litellm_params": {"tags": ["region:eu"]}} + us = {"litellm_params": {"tags": ["region:us"]}} + # required_set={"region:eu"} IS in inherited_required_set -> protected, kept. + result = _trusted_only_pool( + (eu, us), frozenset(), frozenset({"region:eu"}), frozenset(), frozenset({"region:eu"}) + ) + assert result == (eu,) + + +def test_trusted_only_pool_discards_a_value_with_no_inherited_backing_even_if_the_caller_also_sent_it(): + # Regression for the value-collision bypass Greptile and veria-ai both + # flagged: a value with zero inherited backing is discardable even when it + # happens to be the exact value the caller submitted -- there is nothing here + # to distinguish "caller-only" from "caller happened to guess a real policy + # value" at this function's level, which is exactly why protection must be + # keyed off presence in inherited_required_set, never absence from a + # caller-supplied set (see the router-level regression below for the full + # bypass this replaces). + from litellm.router_strategy.tag_based_routing import _trusted_only_pool + + eu = {"litellm_params": {"tags": ["region:eu"]}} + us = {"litellm_params": {"tags": ["region:us"]}} + result = _trusted_only_pool((eu, us), frozenset(), frozenset({"region:eu"}), frozenset(), frozenset()) + assert result == (eu, us) + + +def _eu_region_router(): + # eu-1 deliberately carries no "default" tag, and us-default is the only + # "default"-tagged deployment -- this keeps _default_tagged_pool's outcome a + # single, deterministic deployment id in every scenario below, regardless of + # which of the two candidate pools (trusted-only vs fully-unconstrained) a + # given code path resolves to. + return litellm.Router( + model_list=[ + { + "model_name": "chat", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["region:eu"], + }, + "model_info": {"id": "eu-1", "allow_fail_open": True}, + }, + { + "model_name": "chat", + "litellm_params": { + "model": "gpt-4o-mini", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "tags": ["region:us", "default"], + }, + "model_info": {"id": "us-default", "allow_fail_open": True}, + }, + ], + enable_tag_filtering=True, + ) + + +@pytest.mark.asyncio() +async def test_allow_fail_open_preserves_inherited_constraint_when_caller_tag_causes_exhaustion(): + # ®ion:eu simulates a key/team-inherited hard requirement, captured in + # inherited_tags (a snapshot taken before the caller's own tags are merged + # in); !region:eu simulates the caller's own tag. Combined they exhaust the + # pool (nothing can both carry and not carry region:eu), but allow_fail_open + # must fall back to what still satisfies the inherited requirement, not the + # fully-unconstrained default pool (us-default), and not raise either. + router = _eu_region_router() + + response = await router.acompletion( + model="chat", + messages=[{"role": "user", "content": "hi"}], + metadata={ + "tags": ["®ion:eu", "!region:eu"], + "inherited_tags": ["®ion:eu"], + "caller_tags": ["!region:eu"], + }, + mock_response="hi", + ) + + assert response._hidden_params["model_id"] == "eu-1" + + +@pytest.mark.asyncio() +async def test_allow_fail_open_stays_protected_when_caller_duplicates_the_inherited_tag(): + # Regression for the value-collision bypass Greptile and veria-ai both + # flagged: a caller who resubmits the exact value of an inherited "&" tag + # (here alongside a conflicting "!" for the same value) must not be able to + # strip that value's protection just because it now also appears in + # caller_tags. Protection is keyed off presence in inherited_tags, not + # absence from caller_tags -- if it were the latter, subtracting + # caller_required_set={"region:eu"} from required_set would zero out the + # inherited requirement entirely and this would incorrectly resolve to + # us-default instead of eu-1. + router = _eu_region_router() + + response = await router.acompletion( + model="chat", + messages=[{"role": "user", "content": "hi"}], + metadata={ + "tags": ["®ion:eu", "!region:eu"], + "inherited_tags": ["®ion:eu"], + "caller_tags": ["®ion:eu", "!region:eu"], + }, + mock_response="hi", + ) + + assert response._hidden_params["model_id"] == "eu-1" + + +@pytest.mark.asyncio() +async def test_allow_fail_open_raises_when_inherited_constraint_alone_is_unsatisfiable(): + # Both region:eu and region:us are known to the group (so the unknown-tag + # masking guard does not apply), but no single deployment carries both, and + # inherited_tags confirms the entire required-AND set traces back to policy. + # allow_fail_open must not paper over an inherited requirement that is + # unsatisfiable on its own; it should raise exactly as it would with + # allow_fail_open unset. + router = _eu_region_router() + + with pytest.raises(Exception) as exc_info: + await router.acompletion( + model="chat", + messages=[{"role": "user", "content": "hi"}], + metadata={ + "tags": ["®ion:eu", "®ion:us"], + "inherited_tags": ["®ion:eu", "®ion:us"], + "caller_tags": [], + }, + mock_response="hi", + ) + + from litellm.types.router import RouterErrors + + assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value) + + +@pytest.mark.asyncio() +async def test_allow_fail_open_unconditional_discard_when_inherited_tags_key_absent(): + # No "inherited_tags" key at all (e.g. a direct SDK Router call that never + # went through the proxy's litellm_pre_call_utils.py) must reproduce the exact + # pre-provenance behavior: unconditional fall-open to the default pool, even + # though region:eu here would otherwise look like an inherited requirement. + router = _eu_region_router() + + response = await router.acompletion( + model="chat", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["®ion:eu", "!region:eu"]}, + mock_response="hi", + ) + + assert response._hidden_params["model_id"] == "us-default" + + +# --- tag_routing_prefix must be configurable through every settings-update +# path the router already supports for its sibling enable_tag_filtering, not +# just the config.yaml constructor argument --- + + +def test_router_update_settings_applies_tag_routing_prefix(): + # Regression: tag_routing_prefix was missing from Router.update_settings's + # _allowed_settings, so an operator configuring it via the DB-backed + # router_settings path (proxy_server.py's _add_router_settings_from_db_config, + # which calls update_settings directly) had the value silently ignored. + router = litellm.Router(model_list=[{"model_name": "x", "litellm_params": {"model": "openai/gpt-4o-mini"}}]) + assert router.tag_routing_prefix == "" + + router.update_settings(tag_routing_prefix="route:") + + assert router.tag_routing_prefix == "route:" + + +def test_router_get_settings_includes_tag_routing_prefix(): + router = litellm.Router(model_list=[{"model_name": "x", "litellm_params": {"model": "openai/gpt-4o-mini"}}]) + router.update_settings(tag_routing_prefix="route:") + + assert router.get_settings()["tag_routing_prefix"] == "route:" + + +def test_update_router_config_schema_includes_tag_routing_prefix(): + # The Admin UI's POST /config/update path validates through + # UpdateRouterConfig before calling update_settings; a field missing here + # causes model_dump(exclude_none=True) to silently drop it before + # update_settings is ever called -- the same bug shape LIT-3152 fixed for + # retry_policy (see tests/test_litellm/test_router_retry_policy_update.py). + from litellm.types.router import UpdateRouterConfig + + config = UpdateRouterConfig(tag_routing_prefix="route:") + assert config.model_dump(exclude_none=True)["tag_routing_prefix"] == "route:" diff --git a/type-discipline-budget.json b/type-discipline-budget.json index c990ae52ff2..eee067815d2 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 23057 + "limit": 23039 }, "LIT002": { - "limit": 27156 + "limit": 27154 }, "LIT003": { "limit": 269 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16744 + "limit": 16742 }, "LIT011": { "limit": 5596 diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 35adc9e92cf..05cba547f23 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -34222,6 +34222,8 @@ export interface components { routing_strategy_args?: { [key: string]: unknown; } | null; + /** Tag Routing Prefix */ + tag_routing_prefix?: string | null; /** Timeout */ timeout?: number | null; }; @@ -35391,6 +35393,8 @@ export interface components { }; /** ModelInfo */ litellm__types__router__ModelInfo: { + /** Allow Fail Open */ + allow_fail_open?: boolean | null; /** Base Model */ base_model?: string | null; /** Blocked */ @@ -35410,6 +35414,8 @@ export interface components { * @default false */ db_model: boolean; + /** Enable Tag Filtering */ + enable_tag_filtering?: boolean | null; /** Id */ id: string | null; /** Input Cost Per Character */ From cbf85a015fd1a12f86fa0ef7c1bdbbf82e7007dc Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 11 Aug 2026 11:53:11 -0700 Subject: [PATCH 23/23] feat(proxy): per-key prompt caching toggle via enable_prompt_caching (#36466) * feat(proxy): per-key prompt caching auto-injection via enable_prompt_caching Adds a key-level enable_prompt_caching toggle that auto-injects Anthropic cache_control breakpoints on requests made with that key, without requiring the gateway-wide enable_anthropic_prompt_caching flag. The flag lives in key metadata, is stamped onto the request root by add_key_level_controls, rides kwargs into both the /chat/completions seeding path and the native /v1/messages path, and reuses every existing gate (anthropic/bedrock only, supports_prompt_caching, client markers win). Client-supplied body values are stripped as an untrusted root control field. Includes the Admin UI switch on key create and key edit plus a read-only settings row, and dedupes the key edit view's drifted initial-values objects. * fix(proxy): drop section comment and suppress LIT011 on key-level prompt caching stamp --- .../anthropic_cache_control_hook.py | 30 ++++--- litellm/main.py | 2 + litellm/proxy/_types.py | 2 + litellm/proxy/litellm_pre_call_utils.py | 4 + .../key_management_endpoints.py | 2 + litellm/types/utils.py | 1 + .../test_anthropic_cache_control_hook.py | 81 +++++++++++++++++++ .../test_key_management_endpoints.py | 15 ++++ .../proxy/test_litellm_pre_call_utils.py | 38 +++++++++ .../organisms/create_key_button.tsx | 15 ++++ .../templates/key_edit_view.test.tsx | 30 +++++++ .../components/templates/key_edit_view.tsx | 48 +++++------ .../components/templates/key_info_view.tsx | 7 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 12 +++ 14 files changed, 248 insertions(+), 39 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index f2ef8d63a07..4df6fce74c0 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -382,19 +382,23 @@ class AnthropicCacheControlHook(CustomPromptManagement): model: str, custom_llm_provider: str | None, tools: list | None = None, + enable_prompt_caching: bool | None = None, ) -> list[CacheControlInjectionPoint]: """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on. - Caches the system prompt and the trailing turn, so the stable prefix - (system + tools + history) is reused while the breakpoint advances with - the conversation. Returns [] (stand down) when the flag is off, the - provider does not consume cache_control breakpoints (only anthropic / - bedrock do), the model lacks prompt-caching support, or the request - already carries client-supplied cache_control. + ``enable_prompt_caching`` is the per-request override (stamped from key + metadata by the proxy); True turns auto-injection on for this request + even when the global flag is off. Caches the system prompt and the + trailing turn, so the stable prefix (system + tools + history) is + reused while the breakpoint advances with the conversation. Returns [] + (stand down) when neither flag is on, the provider does not consume + cache_control breakpoints (only anthropic / bedrock do), the model + lacks prompt-caching support, or the request already carries + client-supplied cache_control. """ import litellm - if litellm.enable_anthropic_prompt_caching is not True: + if litellm.enable_anthropic_prompt_caching is not True and enable_prompt_caching is not True: return [] provider = custom_llm_provider @@ -433,6 +437,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): model: str, custom_llm_provider: str | None, tools: list | None = None, + enable_prompt_caching: bool | None = None, ) -> None: """For /chat/completions: resolve the injection points the request should carry. @@ -458,6 +463,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): model=model, custom_llm_provider=custom_llm_provider, tools=tools, + enable_prompt_caching=enable_prompt_caching, ) if points: non_default_params["cache_control_injection_points"] = points @@ -478,12 +484,17 @@ class AnthropicCacheControlHook(CustomPromptManagement): judgment happens once per request; points a prior pass wrote back carry the judged stamp and are never re-judged (see ``_should_stand_down``). When none are configured but - ``litellm.enable_anthropic_prompt_caching`` is on, synthesize default - breakpoints for the native /v1/messages path. Pops the key from kwargs; + ``litellm.enable_anthropic_prompt_caching`` or the per-request + ``enable_prompt_caching`` kwarg (stamped from key metadata) is on, + synthesize default breakpoints for the native /v1/messages path. Pops + both keys from kwargs; if remaining (non-message) points exist they are written back so downstream transforms can handle them. """ typed_messages = cast(list[AllMessageValues], messages) # cast-ok: Anthropic-shaped dicts from v1/messages + enable_prompt_caching: Final = cast( # cast-ok: kwargs is untyped; key stamped as bool by the proxy + bool | None, kwargs.pop("enable_prompt_caching", None) + ) configured: Final = cast( # cast-ok: kwargs is untyped; this key only holds the documented injection-point list list[CacheControlInjectionPoint] | None, kwargs.pop("cache_control_injection_points", None) ) @@ -497,6 +508,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): tools=tools, model=model, custom_llm_provider=custom_llm_provider, + enable_prompt_caching=enable_prompt_caching, ) if not injection_points: return messages, system diff --git a/litellm/main.py b/litellm/main.py index c70a41c891a..bf6ec8004ce 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -504,6 +504,7 @@ async def acompletion( model=model, custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs tools=tools, + enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( @@ -5105,6 +5106,7 @@ def completion( model=model, custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs tools=tools, + enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index dc4f17c7b31..08348187645 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1108,6 +1108,7 @@ class KeyRequestBase(GenerateRequestBase): budget_id: str | None = None tags: list[str] | None = None disable_global_guardrails: bool | None = None + enable_prompt_caching: bool | None = None throttle_on_budget_exceeded: bool | None = None enforced_params: list[str] | None = None allowed_routes: list | None = [] @@ -4124,6 +4125,7 @@ LiteLLM_ManagementEndpoint_MetadataFields: Final = [ "enforced_batch_output_expires_after", "enforced_file_expires_after", "throttle_on_budget_exceeded", + "enable_prompt_caching", ] LiteLLM_ManagementEndpoint_MetadataFields_Premium: Final = [ diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 0924b6aebea..f83061a15ce 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -201,6 +201,7 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = ( "mock_tool_calls", "disable_global_guardrails", "disable_global_guardrail", + "enable_prompt_caching", "opted_out_global_guardrails", "applied_guardrails", "applied_policies", @@ -1333,6 +1334,9 @@ class LiteLLMProxyRequestSetup: if "disable_fallbacks" in key_metadata and isinstance(key_metadata["disable_fallbacks"], bool): data["disable_fallbacks"] = key_metadata["disable_fallbacks"] + if isinstance(key_metadata.get("enable_prompt_caching"), bool): + data["enable_prompt_caching"] = key_metadata["enable_prompt_caching"] # rebind-ok: data is an out-param + ## KEY-LEVEL METADATA data = LiteLLMProxyRequestSetup.add_management_endpoint_metadata_to_request_metadata( data=data, diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 836b223c6ad..ebf44497658 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1593,6 +1593,7 @@ async def generate_key_fn( - policies: Optional[List[str]] - List of policy names to apply to the key. Policies define guardrails, conditions, and inheritance rules. - disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the key. - throttle_on_budget_exceeded: Optional[bool] - When the key exceeds its max_budget, throttle its tpm/rpm to the global budget_exceeded_throttle_percentage instead of blocking the key entirely. + - enable_prompt_caching: Optional[bool] - Auto-inject prompt caching breakpoints (Anthropic cache_control markers) on requests made with this key. Anthropic and Bedrock Claude models only. - permissions: Optional[dict] - key-specific permissions. Currently just used for turning off pii masking (if connected). Example - {"pii": false} - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}}. IF null or {} then no model specific budget. - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. @@ -2693,6 +2694,7 @@ async def update_key_fn( - policies: Optional[List[str]] - List of policy names to apply to the key. Policies define guardrails, conditions, and inheritance rules. - disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the key. - throttle_on_budget_exceeded: Optional[bool] - When the key exceeds its max_budget, throttle its tpm/rpm to the global budget_exceeded_throttle_percentage instead of blocking the key entirely. + - enable_prompt_caching: Optional[bool] - Auto-inject prompt caching breakpoints (Anthropic cache_control markers) on requests made with this key. Anthropic and Bedrock Claude models only. - prompts: Optional[List[str]] - List of prompts that the key is allowed to use. - blocked: Optional[bool] - Whether the key is blocked - aliases: Optional[dict] - Model aliases for the key - [Docs](https://litellm.vercel.app/docs/proxy/virtual_keys#model-aliases) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 74311d59d8e..d4c26df54d8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3467,6 +3467,7 @@ all_litellm_params = ( "caching_groups", "ttl", "cache", + "enable_prompt_caching", "no-log", "base_model", "stream_timeout", diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 47baacd61d7..cc43a424419 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1728,6 +1728,87 @@ class TestEnableAnthropicPromptCaching: assert result_msgs[-1]["content"][-1]["cache_control"] == {"type": "ephemeral"} assert "cache_control" not in result_msgs[0]["content"][-1] + +class TestPerKeyEnablePromptCaching: + """Per-request enable_prompt_caching override (stamped from key metadata) with the global flag off.""" + + MESSAGES: List[AllMessageValues] = [ + {"role": "system", "content": "a long system prompt"}, + {"role": "user", "content": "latest turn"}, + ] + + def _points(self, enable_prompt_caching, model="claude-sonnet-4-5", provider="anthropic", messages=None): + return AnthropicCacheControlHook.get_default_injection_points( + messages=copy.deepcopy(self.MESSAGES) if messages is None else messages, + system=None, + model=model, + custom_llm_provider=provider, + enable_prompt_caching=enable_prompt_caching, + ) + + def test_true_injects_with_global_flag_off(self): + assert litellm.enable_anthropic_prompt_caching is False + assert self._points(True) == [ + {"location": "message", "role": "system", "index": None, "control": {"type": "ephemeral"}}, + {"location": "message", "role": None, "index": -1, "control": {"type": "ephemeral"}}, + ] + + @pytest.mark.parametrize("enable_prompt_caching", [False, None]) + def test_false_and_none_fall_back_to_global_flag(self, enable_prompt_caching): + assert self._points(enable_prompt_caching) == [] + + def test_false_does_not_suppress_global_flag(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert [p["index"] for p in self._points(False)] == [None, -1] + + def test_provider_gate_still_applies(self): + assert self._points(True, model="gpt-4o", provider="openai") == [] + + def test_unsupported_model_gate_still_applies(self): + assert self._points(True, model="anthropic.claude-3-5-sonnet-20240620-v1:0", provider="bedrock") == [] + + def test_client_markers_still_win(self): + messages = [ + {"role": "system", "content": [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": "latest turn"}, + ] + assert self._points(True, messages=messages) == [] + + def test_seed_injects_with_global_flag_off(self): + params: dict = {} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + enable_prompt_caching=True, + ) + assert [p["index"] for p in params["cache_control_injection_points"]] == [None, -1] + + def test_v1_messages_injects_and_pops_flag_from_kwargs(self): + kwargs: dict = {"enable_prompt_caching": True} + result_msgs, result_sys = AnthropicCacheControlHook.maybe_inject_cache_control( + [{"role": "user", "content": [{"type": "text", "text": "latest"}]}], + "a system prompt", + kwargs, + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + assert result_sys == [{"type": "text", "text": "a system prompt", "cache_control": {"type": "ephemeral"}}] + assert result_msgs[-1]["content"][-1]["cache_control"] == {"type": "ephemeral"} + assert "enable_prompt_caching" not in kwargs + + def test_v1_messages_pops_flag_even_when_noop(self): + kwargs: dict = {"enable_prompt_caching": True} + AnthropicCacheControlHook.maybe_inject_cache_control( + [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + None, + kwargs, + model="gpt-4o", + custom_llm_provider="openai", + ) + assert "enable_prompt_caching" not in kwargs + def test_v1_messages_is_noop_when_disabled(self): messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] result_msgs, result_sys = AnthropicCacheControlHook.maybe_inject_cache_control( diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 4c13a3367d9..0a88f59f677 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -1733,6 +1733,21 @@ async def test_update_service_account_works_with_team_id(): await prepare_key_update_data(data=data, existing_key_row=existing_key) +@pytest.mark.asyncio +@pytest.mark.parametrize("flag_value", [True, False]) +async def test_update_key_enable_prompt_caching_folds_into_metadata(flag_value): + """Top-level enable_prompt_caching on /key/update lands in key metadata, including flipping back to False.""" + data = UpdateKeyRequest(key="sk-1", enable_prompt_caching=flag_value) + existing_key = LiteLLM_VerificationToken( + token="hashed", metadata={"enable_prompt_caching": not flag_value} + ) + + updated = await prepare_key_update_data(data=data, existing_key_row=existing_key) + + assert updated["metadata"]["enable_prompt_caching"] is flag_value + assert "enable_prompt_caching" not in {k for k in updated if k != "metadata"} + + @pytest.mark.asyncio async def test_update_preserves_service_account_id_when_metadata_replaced(): """ diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 293cce5fa5d..e31058f402e 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -688,6 +688,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "mock_response": "free response", "mock_tool_calls": [{"id": "call_1"}], "disable_global_guardrails": True, + "enable_prompt_caching": True, "routing_decision": {"cause": "forged", "routed_model": "spoofed"}, "metadata": copy.deepcopy(malicious_metadata), "litellm_metadata": copy.deepcopy(malicious_metadata), @@ -705,6 +706,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): assert "mock_response" not in updated assert "mock_tool_calls" not in updated assert "disable_global_guardrails" not in updated + assert "enable_prompt_caching" not in updated assert "routing_decision" not in updated stripped_keys = { @@ -741,6 +743,42 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): assert "pillar_response_headers" not in snapshot_body["metadata"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key_value, expected", + [(True, True), (False, False), ("yes", None), (None, None)], +) +async def test_key_metadata_enable_prompt_caching_promoted_to_request_root(key_value, expected): + """Key metadata enable_prompt_caching is stamped onto the request root (bools only), even when the client spoofs it.""" + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/v1/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = { + "model": "claude-sonnet-4-5", + "messages": [{"role": "user", "content": "hello"}], + "enable_prompt_caching": "spoofed-by-client", + } + key_metadata = {} if key_value is None else {"enable_prompt_caching": key_value} + + updated = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", metadata=key_metadata), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated.get("enable_prompt_caching") == expected + + @pytest.mark.asyncio @pytest.mark.parametrize( "control_field", diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 19abae73549..8951a471f84 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -1191,6 +1191,21 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp > + + Enable Prompt Caching{" "} + + + + + } + name="enable_prompt_caching" + valuePropName="checked" + > + + diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx index 29b32c1f7fc..fb8ab87e45f 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx @@ -410,6 +410,36 @@ describe("KeyEditView", () => { }); }); + it("should initialize and submit enable_prompt_caching from key metadata", async () => { + const onSubmitMock = vi.fn().mockResolvedValue(undefined); + const keyDataWithPromptCaching = { + ...MOCK_KEY_DATA, + metadata: { ...MOCK_KEY_DATA.metadata, enable_prompt_caching: true }, + }; + + renderWithProviders( + {}} + onSubmit={onSubmitMock} + accessToken={"test-token"} + userID={"test-user"} + userRole={"admin"} + premiumUser={false} + />, + ); + + await waitFor(() => { + expect(screen.getByText("Enable Prompt Caching")).toBeInTheDocument(); + }); + + await userEvent.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(onSubmitMock).toHaveBeenCalledWith(expect.objectContaining({ enable_prompt_caching: true })); + }); + }); + it("should disable models field when management routes are selected", async () => { const keyDataWithManagementRoutes = { ...MOCK_KEY_DATA, diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index e407527f562..36dc02f7bd0 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -150,6 +150,7 @@ export function KeyEditView({ guardrails: keyData.metadata?.guardrails, disable_global_guardrails: keyData.metadata?.disable_global_guardrails || false, throttle_on_budget_exceeded: keyData.metadata?.throttle_on_budget_exceeded || false, + enable_prompt_caching: keyData.metadata?.enable_prompt_caching || false, ...estimateFields(keyData.metadata), prompts: keyData.metadata?.prompts, tags: keyData.metadata?.tags, @@ -178,36 +179,8 @@ export function KeyEditView({ }; useEffect(() => { - form.setFieldsValue({ - ...keyData, - token: keyData.token || keyData.token_id, - budget_duration: canonicalBudgetDuration(keyData.budget_duration), - metadata: formatMetadataForDisplay(stripTagsFromMetadata(keyData.metadata)), - guardrails: keyData.metadata?.guardrails, - disable_global_guardrails: keyData.metadata?.disable_global_guardrails || false, - prompts: keyData.metadata?.prompts, - tags: keyData.metadata?.tags, - vector_stores: keyData.object_permission?.vector_stores || [], - mcp_servers_and_groups: { - servers: keyData.object_permission?.mcp_servers || [], - accessGroups: keyData.object_permission?.mcp_access_groups || [], - toolsets: keyData.object_permission?.mcp_toolsets || [], - }, - mcp_tool_permissions: keyData.object_permission?.mcp_tool_permissions || {}, - throttle_on_budget_exceeded: keyData.metadata?.throttle_on_budget_exceeded || false, - ...estimateFields(keyData.metadata), - logging_settings: extractLoggingSettings(keyData.metadata), - disabled_callbacks: Array.isArray(keyData.metadata?.litellm_disabled_callbacks) - ? mapInternalToDisplayNames(keyData.metadata.litellm_disabled_callbacks) - : [], - access_group_ids: keyData.access_group_ids || [], - auto_rotate: keyData.auto_rotate || false, - ...(keyData.rotation_interval && { rotation_interval: keyData.rotation_interval }), - allowed_routes: - Array.isArray(keyData.allowed_routes) && keyData.allowed_routes.length > 0 - ? keyData.allowed_routes.join(", ") - : "", - }); + form.setFieldsValue(initialValues); + // eslint-disable-next-line react-hooks/exhaustive-deps -- initialValues is rebuilt from keyData every render; depending on it would re-run each render }, [keyData, form]); // Sync auto-rotation state with form values @@ -532,6 +505,21 @@ export function KeyEditView({ + + Enable Prompt Caching{" "} + + + + + } + name="enable_prompt_caching" + valuePropName="checked" + > + + + diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 6a2a4a88425..6fd6547a995 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -782,6 +782,13 @@ export default function KeyInfoView({ {currentKeyData.expires ? formatTimestamp(currentKeyData.expires) : "Never"} + {Boolean(currentKeyData.metadata?.enable_prompt_caching) && ( +
+ Prompt Caching + Enabled (auto-injects cache_control markers on Anthropic and Bedrock Claude requests) +
+ )} +