From eef117bb790ec3ac347585b6ec90a6bdfcbd5626 Mon Sep 17 00:00:00 2001 From: fzowl Date: Fri, 27 Dec 2024 22:04:02 +0100 Subject: [PATCH 001/126] Refresh VoyageAI models and prices and context --- model_prices_and_context_window.json | 94 ++++++++++++++++++---------- 1 file changed, 62 insertions(+), 32 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5f98c0e68b0..a12d114f952 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -7566,30 +7566,6 @@ "litellm_provider": "cloudflare", "mode": "chat" }, - "voyage/voyage-01": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, - "litellm_provider": "voyage", - "mode": "embedding" - }, - "voyage/voyage-lite-01": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, - "litellm_provider": "voyage", - "mode": "embedding" - }, - "voyage/voyage-large-2": { - "max_tokens": 16000, - "max_input_tokens": 16000, - "input_cost_per_token": 0.00000012, - "output_cost_per_token": 0.000000, - "litellm_provider": "voyage", - "mode": "embedding" - }, "voyage/voyage-law-2": { "max_tokens": 16000, "max_input_tokens": 16000, @@ -7614,14 +7590,6 @@ "litellm_provider": "voyage", "mode": "embedding" }, - "voyage/voyage-lite-02-instruct": { - "max_tokens": 4000, - "max_input_tokens": 4000, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, - "litellm_provider": "voyage", - "mode": "embedding" - }, "voyage/voyage-finance-2": { "max_tokens": 4000, "max_input_tokens": 4000, @@ -7630,6 +7598,68 @@ "litellm_provider": "voyage", "mode": "embedding" }, + "voyage/voyage-3-large": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "input_cost_per_token": 0.00000018, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/voyage-3": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "input_cost_per_token": 0.00000006, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/voyage-3-lite": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "input_cost_per_token": 0.00000002, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/voyage-code-3": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "input_cost_per_token": 0.00000018, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/voyage-multimodal-3": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "input_cost_per_token": 0.00000012, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/rerank-2": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "max_output_tokens": 16000, + "max_query_tokens": 16000, + "input_cost_per_token": 0.00000005, + "input_cost_per_query": 0.00000005, + "output_cost_per_token": 0.0, + "litellm_provider": "voyage", + "mode": "rerank" + }, + "voyage/rerank-2-lite": { + "max_tokens": 8000, + "max_input_tokens": 8000, + "max_output_tokens": 8000, + "max_query_tokens": 8000, + "input_cost_per_token": 0.00000002, + "input_cost_per_query": 0.00000002, + "output_cost_per_token": 0.0, + "litellm_provider": "voyage", + "mode": "rerank" + }, "databricks/databricks-meta-llama-3-1-405b-instruct": { "max_tokens": 128000, "max_input_tokens": 128000, From 6876a14838605865276ac17ea9991e1874f8d64a Mon Sep 17 00:00:00 2001 From: fzowl Date: Mon, 30 Dec 2024 11:50:37 +0100 Subject: [PATCH 002/126] Refresh VoyageAI models and prices and context --- model_prices_and_context_window.json | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2ff5e40247d..0701b20af49 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -7613,6 +7613,38 @@ "litellm_provider": "cloudflare", "mode": "chat" }, + "voyage/voyage-01": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/voyage-lite-01": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/voyage-large-2": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "input_cost_per_token": 0.00000012, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/voyage-lite-02-instruct": { + "max_tokens": 4000, + "max_input_tokens": 4000, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, "voyage/voyage-law-2": { "max_tokens": 16000, "max_input_tokens": 16000, From bb0f74c48f9a85874f2f104a6de0f043d4658ea6 Mon Sep 17 00:00:00 2001 From: fzowl Date: Mon, 30 Dec 2024 11:54:37 +0100 Subject: [PATCH 003/126] Refresh VoyageAI models and prices and context --- model_prices_and_context_window.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0701b20af49..1a9ed533e38 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -7637,6 +7637,14 @@ "litellm_provider": "voyage", "mode": "embedding" }, + "voyage/voyage-finance-2": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "input_cost_per_token": 0.00000012, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, "voyage/voyage-lite-02-instruct": { "max_tokens": 4000, "max_input_tokens": 4000, @@ -7669,14 +7677,6 @@ "litellm_provider": "voyage", "mode": "embedding" }, - "voyage/voyage-finance-2": { - "max_tokens": 4000, - "max_input_tokens": 4000, - "input_cost_per_token": 0.00000012, - "output_cost_per_token": 0.000000, - "litellm_provider": "voyage", - "mode": "embedding" - }, "voyage/voyage-3-large": { "max_tokens": 32000, "max_input_tokens": 32000, From 9690cb66c6e51f81db586152d822965e1db94a60 Mon Sep 17 00:00:00 2001 From: fzowl Date: Mon, 3 Feb 2025 15:21:20 +0100 Subject: [PATCH 004/126] Updating the available VoyageAI models in the docs --- docs/my-website/docs/providers/voyage.md | 27 +++++++++++++++--------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/my-website/docs/providers/voyage.md b/docs/my-website/docs/providers/voyage.md index a56a1408ea9..6ab6b1846f5 100644 --- a/docs/my-website/docs/providers/voyage.md +++ b/docs/my-website/docs/providers/voyage.md @@ -14,7 +14,7 @@ import os os.environ['VOYAGE_API_KEY'] = "" response = embedding( - model="voyage/voyage-01", + model="voyage/voyage-3-large", input=["good morning from litellm"], ) print(response) @@ -23,13 +23,20 @@ print(response) ## Supported Models All models listed here https://docs.voyageai.com/embeddings/#models-and-specifics are supported -| Model Name | Function Call | -|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| voyage-2 | `embedding(model="voyage/voyage-2", input)` | -| voyage-large-2 | `embedding(model="voyage/voyage-large-2", input)` | -| voyage-law-2 | `embedding(model="voyage/voyage-law-2", input)` | -| voyage-code-2 | `embedding(model="voyage/voyage-code-2", input)` | +| Model Name | Function Call | +|-------------------------|------------------------------------------------------------| +| voyage-3-large | `embedding(model="voyage/voyage-3-large", input)` | +| voyage-3 | `embedding(model="voyage/voyage-3", input)` | +| voyage-3-lite | `embedding(model="voyage/voyage-3-lite", input)` | +| voyage-code-3 | `embedding(model="voyage/voyage-code-3", input)` | +| voyage-finance-2 | `embedding(model="voyage/voyage-finance-2", input)` | +| voyage-law-2 | `embedding(model="voyage/voyage-law-2", input)` | +| voyage-code-2 | `embedding(model="voyage/voyage-code-2", input)` | +| voyage-multilingual-2 | `embedding(model="voyage/voyage-multilingual-2 ", input)` | +| voyage-large-2-instruct | `embedding(model="voyage/voyage-large-2-instruct", input)` | +| voyage-large-2 | `embedding(model="voyage/voyage-large-2", input)` | +| voyage-2 | `embedding(model="voyage/voyage-2", input)` | | voyage-lite-02-instruct | `embedding(model="voyage/voyage-lite-02-instruct", input)` | -| voyage-01 | `embedding(model="voyage/voyage-01", input)` | -| voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` | -| voyage-lite-01-instruct | `embedding(model="voyage/voyage-lite-01-instruct", input)` | \ No newline at end of file +| voyage-01 | `embedding(model="voyage/voyage-01", input)` | +| voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` | +| voyage-lite-01-instruct | `embedding(model="voyage/voyage-lite-01-instruct", input)` | From 818db5e59032f37bd8f8440fd064607ae3f4c924 Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 21 May 2025 13:09:42 +0200 Subject: [PATCH 005/126] Updating the available VoyageAI models in the docs --- docs/my-website/docs/providers/voyage.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/providers/voyage.md b/docs/my-website/docs/providers/voyage.md index 6ab6b1846f5..4b729bc9f58 100644 --- a/docs/my-website/docs/providers/voyage.md +++ b/docs/my-website/docs/providers/voyage.md @@ -25,6 +25,8 @@ All models listed here https://docs.voyageai.com/embeddings/#models-and-specific | Model Name | Function Call | |-------------------------|------------------------------------------------------------| +| voyage-3.5 | `embedding(model="voyage/voyage-3.5", input)` | +| voyage-3.5-lite | `embedding(model="voyage/voyage-3.5-lite", input)` | | voyage-3-large | `embedding(model="voyage/voyage-3-large", input)` | | voyage-3 | `embedding(model="voyage/voyage-3", input)` | | voyage-3-lite | `embedding(model="voyage/voyage-3-lite", input)` | @@ -35,8 +37,8 @@ All models listed here https://docs.voyageai.com/embeddings/#models-and-specific | voyage-multilingual-2 | `embedding(model="voyage/voyage-multilingual-2 ", input)` | | voyage-large-2-instruct | `embedding(model="voyage/voyage-large-2-instruct", input)` | | voyage-large-2 | `embedding(model="voyage/voyage-large-2", input)` | -| voyage-2 | `embedding(model="voyage/voyage-2", input)` | +| voyage-2 | `embedding(model="voyage/voyage-2", input)` | | voyage-lite-02-instruct | `embedding(model="voyage/voyage-lite-02-instruct", input)` | -| voyage-01 | `embedding(model="voyage/voyage-01", input)` | -| voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` | +| voyage-01 | `embedding(model="voyage/voyage-01", input)` | +| voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` | | voyage-lite-01-instruct | `embedding(model="voyage/voyage-lite-01-instruct", input)` | From f88dd65590460df01c49a6a58ab95ee7f28d8b58 Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 29 Jul 2026 14:31:25 +0200 Subject: [PATCH 006/126] feat(voyage): add voyage-4 family + voyage-context-4, fix contextual list[str] input - Add voyage-4, voyage-4-large, voyage-4-lite, voyage-4-nano (/bin/bash) and voyage-context-4 to the cost/context map (both root and backup json). - Contextual embeddings: normalize flat list[str] input. Voyage rejects a flat list[str] unless input_type='query', so wrap each string as its own single-chunk document otherwise; single str -> [[str]]; list[list[str]] passes through. Keeps list[str] when input_type='query'. - Add tests for input normalization and voyage-4 family pricing. --- .../embedding/transformation_contextual.py | 33 ++++++- ...odel_prices_and_context_window_backup.json | 40 +++++++++ model_prices_and_context_window.json | 40 +++++++++ tests/llm_translation/test_voyage_ai.py | 85 +++++++++++++++++++ 4 files changed, 197 insertions(+), 1 deletion(-) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index 4df2fa4ba31..2d661610eaa 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -106,11 +106,42 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): headers: dict, ) -> dict: return { - "inputs": input, + "inputs": self._transform_contextual_inputs(input, optional_params), "model": model, **optional_params, } + @staticmethod + def _transform_contextual_inputs( + input: Union[AllEmbeddingInputValues, List[List[str]]], + optional_params: dict, + ) -> List[List[str]]: + """ + Voyage's contextualized embeddings API expects ``inputs`` to be a + ``list[list[str]]`` (each inner list is a document made of chunks that + share context). + + It also accepts a flat ``list[str]`` - but only when + ``input_type == "query"``. In every other case a flat list must be + wrapped so each string becomes its own single-chunk document, otherwise + the API rejects the request with a 400. + + Reference: https://docs.voyageai.com/reference/contextualized-embeddings-api + """ + # Single string -> one document with a single chunk. + if isinstance(input, str): + return [[input]] + + # Flat list[str]: keep as list[str] when the API allows it + # (input_type="query"), otherwise wrap each string as its own document. + if isinstance(input, list) and all(isinstance(i, str) for i in input): + if optional_params.get("input_type") == "query": + return input # type: ignore[return-value] + return [[i] for i in input] + + # Already list[list[str]] (or another shape) -> pass through unchanged. + return input # type: ignore[return-value] + def transform_embedding_response( self, model: str, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c584deb683a..fc710497592 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -27345,6 +27345,38 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, + "voyage/voyage-4": { + "input_cost_per_token": 6e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "voyage/voyage-4-large": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "voyage/voyage-4-lite": { + "input_cost_per_token": 2e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "voyage/voyage-4-nano": { + "input_cost_per_token": 0.0, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, "voyage/voyage-code-2": { "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", @@ -27369,6 +27401,14 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, + "voyage/voyage-context-4": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "voyage", + "max_input_tokens": 120000, + "max_tokens": 120000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, "voyage/voyage-finance-2": { "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c584deb683a..fc710497592 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -27345,6 +27345,38 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, + "voyage/voyage-4": { + "input_cost_per_token": 6e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "voyage/voyage-4-large": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "voyage/voyage-4-lite": { + "input_cost_per_token": 2e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "voyage/voyage-4-nano": { + "input_cost_per_token": 0.0, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, "voyage/voyage-code-2": { "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", @@ -27369,6 +27401,14 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, + "voyage/voyage-context-4": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "voyage", + "max_input_tokens": 120000, + "max_tokens": 120000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, "voyage/voyage-finance-2": { "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", diff --git a/tests/llm_translation/test_voyage_ai.py b/tests/llm_translation/test_voyage_ai.py index a0b9ee0a44b..577ee808035 100644 --- a/tests/llm_translation/test_voyage_ai.py +++ b/tests/llm_translation/test_voyage_ai.py @@ -71,6 +71,26 @@ class TestVoyageAI(BaseLLMEmbeddingTest): assert response.usage.total_tokens > 0 +@pytest.mark.parametrize( + "model, expected_input_cost", + [ + ("voyage/voyage-4", 6e-08), + ("voyage/voyage-4-large", 1.2e-07), + ("voyage/voyage-4-lite", 2e-08), + ("voyage/voyage-4-nano", 0.0), + ("voyage/voyage-context-4", 1.2e-07), + ], +) +def test_voyage_4_family_pricing_registered(model, expected_input_cost): + """The voyage-4 family and voyage-context-4 must be in the cost map.""" + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + info = litellm.get_model_info(model=model) + assert info["litellm_provider"] == "voyage" + assert info["mode"] == "embedding" + assert info["input_cost_per_token"] == expected_input_cost + + def test_voyage_ai_embedding_extra_params(): """Test Voyage AI embedding with extra parameters""" try: @@ -142,6 +162,7 @@ class TestVoyageContextualEmbeddings: # Test contextual model detection assert config.is_contextualized_embeddings("voyage-context-3") is True + assert config.is_contextualized_embeddings("voyage-context-4") is True assert config.is_contextualized_embeddings("voyage-context-2") is True assert config.is_contextualized_embeddings("context-model") is True @@ -198,6 +219,70 @@ class TestVoyageContextualEmbeddings: assert transformed["model"] == "voyage-context-3" assert transformed["encoding_format"] == "float" + def test_contextual_flat_list_str_is_wrapped(self): + """Flat list[str] without input_type must be wrapped to list[list[str]].""" + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + + transformed = config.transform_embedding_request( + "voyage-context-4", ["Hello", "world"], {}, {} + ) + + # Voyage rejects a flat list[str] unless input_type="query", so each + # string becomes its own single-chunk document. + assert transformed["inputs"] == [["Hello"], ["world"]] + assert transformed["model"] == "voyage-context-4" + + def test_contextual_flat_list_str_query_stays_flat(self): + """Flat list[str] with input_type='query' is sent as-is (list[str]).""" + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + + transformed = config.transform_embedding_request( + "voyage-context-4", + ["Hello", "world"], + {"input_type": "query"}, + {}, + ) + + assert transformed["inputs"] == ["Hello", "world"] + assert transformed["input_type"] == "query" + + def test_contextual_single_string_is_wrapped(self): + """A single string is wrapped to a single document with a single chunk.""" + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + + transformed = config.transform_embedding_request( + "voyage-context-4", "Hello", {}, {} + ) + + assert transformed["inputs"] == [["Hello"]] + + def test_contextual_nested_input_passthrough(self): + """Already-nested list[list[str]] input is passed through unchanged.""" + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + + nested = [["Hello", "world"], ["Test"]] + transformed = config.transform_embedding_request( + "voyage-context-4", nested, {}, {} + ) + + assert transformed["inputs"] == nested + def test_contextual_embedding_response_transformation(self): """Test response transformation for contextual embeddings""" from litellm.llms.voyage.embedding.transformation_contextual import ( From 328b54d0042d213aee3e1e2c025480eae5a214d8 Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 29 Jul 2026 15:22:31 +0200 Subject: [PATCH 007/126] fix(voyage): auto-chunk flat list[str] for contextual embeddings; fix mypy Rework per review: - Contextual inputs now prefer a flat list[str] and let the API auto-chunk (enable_auto_chunking=True, chunk_size=32000, input_type=document) instead of pre-wrapping into list[list[str]]. Matches the live API contract: * flat list[str] + input_type=query -> kept flat * flat list[str] otherwise / single str -> flat + auto-chunking * list[list[str]] -> passthrough - Fix mypy list-item error from the previous wrapping approach. - Update tests for the new normalization; all pass. --- .../embedding/transformation_contextual.py | 73 ++++++++++++++----- tests/llm_translation/test_voyage_ai.py | 47 ++++++++++-- 2 files changed, 94 insertions(+), 26 deletions(-) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index 2d661610eaa..d555a1a2266 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -2,7 +2,7 @@ This module is used to transform the request and response for the Voyage contextualized embeddings API. This would be used for all the contextualized embeddings models in Voyage. """ -from typing import List, Optional, Union +from typing import Any, Dict, List, Optional, Tuple, Union import httpx @@ -98,6 +98,11 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): "Authorization": f"Bearer {api_key}", } + # Chunk size (in tokens) used when the API auto-chunks a flat ``list[str]``. + # Matches the voyage-context-4 context window so each string stays a single + # chunk instead of being split. + AUTO_CHUNK_SIZE = 32000 + def transform_embedding_request( self, model: str, @@ -105,42 +110,74 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): optional_params: dict, headers: dict, ) -> dict: + inputs, extra_params = self._transform_contextual_inputs(input, optional_params) return { - "inputs": self._transform_contextual_inputs(input, optional_params), + "inputs": inputs, "model": model, **optional_params, + **extra_params, } - @staticmethod + @classmethod def _transform_contextual_inputs( + cls, input: Union[AllEmbeddingInputValues, List[List[str]]], optional_params: dict, - ) -> List[List[str]]: + ) -> Tuple[Union[List[str], List[List[str]]], dict]: """ - Voyage's contextualized embeddings API expects ``inputs`` to be a - ``list[list[str]]`` (each inner list is a document made of chunks that - share context). + Normalize ``input`` for Voyage's contextualized embeddings API and + return ``(inputs, extra_params)`` where ``extra_params`` carries any + request fields (e.g. auto-chunking) needed for the chosen shape. - It also accepts a flat ``list[str]`` - but only when - ``input_type == "query"``. In every other case a flat list must be - wrapped so each string becomes its own single-chunk document, otherwise - the API rejects the request with a 400. + The API contract (verified against the live endpoint) is: + + - A flat ``list[str]`` is only accepted with ``input_type="query"`` or + with ``enable_auto_chunking=True`` (which itself requires + ``input_type="document"``). + - A ``list[list[str]]`` (each inner list = one document's chunks) is + always accepted. + + So we prefer to send a flat ``list[str]`` and let the API auto-chunk, + instead of pre-wrapping into ``list[list[str]]``: + + - ``str`` -> ``[str]`` + ``enable_auto_chunking`` (input_type=document) + - flat ``list[str]`` + ``input_type="query"`` -> kept flat, as-is + - flat ``list[str]`` otherwise -> kept flat + ``enable_auto_chunking`` + (input_type=document) + - ``list[list[str]]`` -> passed through unchanged Reference: https://docs.voyageai.com/reference/contextualized-embeddings-api """ - # Single string -> one document with a single chunk. + # Single string -> a one-element flat list, auto-chunked. if isinstance(input, str): - return [[input]] + return [input], cls._auto_chunk_params(optional_params) - # Flat list[str]: keep as list[str] when the API allows it - # (input_type="query"), otherwise wrap each string as its own document. + # Flat list[str]. if isinstance(input, list) and all(isinstance(i, str) for i in input): if optional_params.get("input_type") == "query": - return input # type: ignore[return-value] - return [[i] for i in input] + # The API accepts a flat query list as-is. + return input, {} # type: ignore[return-value] + # Otherwise let the API auto-chunk the flat list. + return input, cls._auto_chunk_params(optional_params) # type: ignore[return-value] # Already list[list[str]] (or another shape) -> pass through unchanged. - return input # type: ignore[return-value] + return input, {} # type: ignore[return-value] + + @classmethod + def _auto_chunk_params(cls, optional_params: dict) -> dict: + """ + Params required to send a flat ``list[str]`` to the contextualized API. + + ``enable_auto_chunking=True`` requires ``input_type="document"``, so set + it unless the caller already provided an ``input_type``. + """ + params: Dict[str, Any] = { + "enable_auto_chunking": True, + "chunk_size": cls.AUTO_CHUNK_SIZE, + } + if not optional_params.get("input_type"): + params["input_type"] = "document" + return params def transform_embedding_response( self, diff --git a/tests/llm_translation/test_voyage_ai.py b/tests/llm_translation/test_voyage_ai.py index 577ee808035..b7a2c41a979 100644 --- a/tests/llm_translation/test_voyage_ai.py +++ b/tests/llm_translation/test_voyage_ai.py @@ -219,8 +219,13 @@ class TestVoyageContextualEmbeddings: assert transformed["model"] == "voyage-context-3" assert transformed["encoding_format"] == "float" - def test_contextual_flat_list_str_is_wrapped(self): - """Flat list[str] without input_type must be wrapped to list[list[str]].""" + def test_contextual_flat_list_str_is_auto_chunked(self): + """Flat list[str] without input_type stays flat and is auto-chunked. + + The API accepts a flat list[str] only with input_type="query" or with + enable_auto_chunking=True (which requires input_type="document"), so we + keep the list flat and let the API auto-chunk it. + """ from litellm.llms.voyage.embedding.transformation_contextual import ( VoyageContextualEmbeddingConfig, ) @@ -231,10 +236,11 @@ class TestVoyageContextualEmbeddings: "voyage-context-4", ["Hello", "world"], {}, {} ) - # Voyage rejects a flat list[str] unless input_type="query", so each - # string becomes its own single-chunk document. - assert transformed["inputs"] == [["Hello"], ["world"]] + assert transformed["inputs"] == ["Hello", "world"] assert transformed["model"] == "voyage-context-4" + assert transformed["enable_auto_chunking"] is True + assert transformed["chunk_size"] == 32000 + assert transformed["input_type"] == "document" def test_contextual_flat_list_str_query_stays_flat(self): """Flat list[str] with input_type='query' is sent as-is (list[str]).""" @@ -253,9 +259,31 @@ class TestVoyageContextualEmbeddings: assert transformed["inputs"] == ["Hello", "world"] assert transformed["input_type"] == "query" + # A query list is accepted as-is, no auto-chunking needed. + assert "enable_auto_chunking" not in transformed - def test_contextual_single_string_is_wrapped(self): - """A single string is wrapped to a single document with a single chunk.""" + def test_contextual_flat_list_str_document_input_type_preserved(self): + """Explicit input_type='document' is preserved while auto-chunking.""" + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + + transformed = config.transform_embedding_request( + "voyage-context-4", + ["Hello", "world"], + {"input_type": "document"}, + {}, + ) + + assert transformed["inputs"] == ["Hello", "world"] + assert transformed["input_type"] == "document" + assert transformed["enable_auto_chunking"] is True + assert transformed["chunk_size"] == 32000 + + def test_contextual_single_string_is_auto_chunked(self): + """A single string becomes a one-element flat list and is auto-chunked.""" from litellm.llms.voyage.embedding.transformation_contextual import ( VoyageContextualEmbeddingConfig, ) @@ -266,7 +294,10 @@ class TestVoyageContextualEmbeddings: "voyage-context-4", "Hello", {}, {} ) - assert transformed["inputs"] == [["Hello"]] + assert transformed["inputs"] == ["Hello"] + assert transformed["enable_auto_chunking"] is True + assert transformed["chunk_size"] == 32000 + assert transformed["input_type"] == "document" def test_contextual_nested_input_passthrough(self): """Already-nested list[list[str]] input is passed through unchanged.""" From be6487d33c43dcf28de410fe209ede88712513b8 Mon Sep 17 00:00:00 2001 From: fzowl <160063452+fzowl@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:20:27 +0200 Subject: [PATCH 008/126] Update litellm/llms/voyage/embedding/transformation_contextual.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/llms/voyage/embedding/transformation_contextual.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index 84ed856e080..c4eb6f34283 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -148,6 +148,8 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): """ # Single string -> a one-element flat list, auto-chunked. if isinstance(input, str): + if optional_params.get("input_type") == "query": + return [input], {} return [input], cls._auto_chunk_params(optional_params) # Flat list[str]. From e8ae29f135d35bbea296d4fa935ef1820c65bb0d Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 29 Jul 2026 17:45:08 +0200 Subject: [PATCH 009/126] fix(voyage): ruff format + add missing query-string test coverage --- .../embedding/transformation_contextual.py | 1 + tests/llm_translation/test_voyage_ai.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index c4eb6f34283..4d8bcc7ff80 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -2,6 +2,7 @@ This module is used to transform the request and response for the Voyage contextualized embeddings API. This would be used for all the contextualized embeddings models in Voyage. """ + from typing import Any, Dict, List, Optional, Tuple, Union import httpx diff --git a/tests/llm_translation/test_voyage_ai.py b/tests/llm_translation/test_voyage_ai.py index a0d291bc02f..e6c73240a0e 100644 --- a/tests/llm_translation/test_voyage_ai.py +++ b/tests/llm_translation/test_voyage_ai.py @@ -300,6 +300,22 @@ class TestVoyageContextualEmbeddings: assert transformed["chunk_size"] == 32000 assert transformed["input_type"] == "document" + def test_contextual_single_string_query_no_auto_chunk(self): + """A single string with input_type='query' is not auto-chunked.""" + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + + transformed = config.transform_embedding_request( + "voyage-context-4", "Hello", {"input_type": "query"}, {} + ) + + assert transformed["inputs"] == ["Hello"] + assert transformed["input_type"] == "query" + assert "enable_auto_chunking" not in transformed + def test_contextual_nested_input_passthrough(self): """Already-nested list[list[str]] input is passed through unchanged.""" from litellm.llms.voyage.embedding.transformation_contextual import ( From bb6db24d58bb93454a77bbcb283f1f6ad4953a1d Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 29 Jul 2026 18:37:25 +0200 Subject: [PATCH 010/126] fix(voyage): use modern type annotations to satisfy strict ruff budget (TID251, UP006) --- .../embedding/transformation_contextual.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index 4d8bcc7ff80..8f029c27be7 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -3,7 +3,9 @@ This module is used to transform the request and response for the Voyage context This would be used for all the contextualized embeddings models in Voyage. """ -from typing import Any, Dict, List, Optional, Tuple, Union +from __future__ import annotations + +from typing import Any import httpx @@ -20,7 +22,7 @@ class VoyageError(BaseLLMException): self, status_code: int, message: str, - headers: Union[dict, httpx.Headers] = {}, + headers: dict | httpx.Headers = {}, ): self.status_code = status_code self.message = message @@ -43,12 +45,12 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): def get_complete_url( self, - api_base: Optional[str], - api_key: Optional[str], + api_base: str | None, + api_key: str | None, model: str, optional_params: dict, litellm_params: dict, - stream: Optional[bool] = None, + stream: bool | None = None, ) -> str: if api_base: if not api_base.endswith("/contextualizedembeddings"): @@ -81,11 +83,11 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): self, headers: dict, model: str, - messages: List[AllMessageValues], + messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - api_key: Optional[str] = None, - api_base: Optional[str] = None, + api_key: str | None = None, + api_base: str | None = None, ) -> dict: if api_key is None: api_key = ( @@ -105,7 +107,7 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): def transform_embedding_request( self, model: str, - input: Union[AllEmbeddingInputValues, List[List[str]]], + input: AllEmbeddingInputValues | list[list[str]], optional_params: dict, headers: dict, ) -> dict: @@ -120,9 +122,9 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): @classmethod def _transform_contextual_inputs( cls, - input: Union[AllEmbeddingInputValues, List[List[str]]], + input: AllEmbeddingInputValues | list[list[str]], optional_params: dict, - ) -> Tuple[Union[List[str], List[List[str]]], dict]: + ) -> tuple[list[str] | list[list[str]], dict]: """ Normalize ``input`` for Voyage's contextualized embeddings API and return ``(inputs, extra_params)`` where ``extra_params`` carries any @@ -172,7 +174,7 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): ``enable_auto_chunking=True`` requires ``input_type="document"``, so set it unless the caller already provided an ``input_type``. """ - params: Dict[str, Any] = { + params: dict[str, Any] = { "enable_auto_chunking": True, "chunk_size": cls.AUTO_CHUNK_SIZE, } @@ -186,7 +188,7 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): raw_response: httpx.Response, model_response: EmbeddingResponse, logging_obj: LiteLLMLoggingObj, - api_key: Optional[str] = None, + api_key: str | None = None, request_data: dict = {}, optional_params: dict = {}, litellm_params: dict = {}, @@ -208,9 +210,7 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): model_response.usage = usage return model_response - def get_error_class( - self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] - ) -> BaseLLMException: + def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: return VoyageError(message=error_message, status_code=status_code, headers=headers) @staticmethod From eddb2225dc6dbc79238f4e6a7815c40cb7957a6d Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 29 Jul 2026 21:29:38 +0200 Subject: [PATCH 011/126] fix(voyage): add mutable-ok pragmas and replace type: ignore with pyright: ignore (LIT001, LIT009) --- .../embedding/transformation_contextual.py | 84 ++++++++++--------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index 8f029c27be7..04cb58bcfcc 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -22,11 +22,14 @@ class VoyageError(BaseLLMException): self, status_code: int, message: str, - headers: dict | httpx.Headers = {}, + headers: dict | httpx.Headers = {}, # mutable-ok: base class signature ): self.status_code = status_code self.message = message - self.request = httpx.Request(method="POST", url="https://api.voyageai.com/v1/contextualizedembeddings") + self.request = httpx.Request( + method="POST", + url="https://api.voyageai.com/v1/contextualizedembeddings", + ) self.response = httpx.Response(status_code=status_code, request=self.request) super().__init__( status_code=status_code, @@ -48,8 +51,8 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): api_base: str | None, api_key: str | None, model: str, - optional_params: dict, - litellm_params: dict, + optional_params: dict, # mutable-ok: base class signature + litellm_params: dict, # mutable-ok: base class signature stream: bool | None = None, ) -> str: if api_base: @@ -58,16 +61,16 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): return api_base return "https://api.voyageai.com/v1/contextualizedembeddings" - def get_supported_openai_params(self, model: str) -> list: + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: base class signature return ["encoding_format", "dimensions"] def map_openai_params( self, - non_default_params: dict, - optional_params: dict, + non_default_params: dict, # mutable-ok: base class signature + optional_params: dict, # mutable-ok: base class signature model: str, drop_params: bool, - ) -> dict: + ) -> dict: # mutable-ok: base class signature """ Map OpenAI params to Voyage params @@ -81,14 +84,14 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): def validate_environment( self, - headers: dict, + headers: dict, # mutable-ok: base class signature model: str, - messages: list[AllMessageValues], - optional_params: dict, - litellm_params: dict, + messages: list[AllMessageValues], # mutable-ok: base class signature + optional_params: dict, # mutable-ok: base class signature + litellm_params: dict, # mutable-ok: base class signature api_key: str | None = None, api_base: str | None = None, - ) -> dict: + ) -> dict: # mutable-ok: base class signature if api_key is None: api_key = ( get_secret_str("VOYAGE_API_KEY") @@ -99,18 +102,15 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): "Authorization": f"Bearer {api_key}", } - # Chunk size (in tokens) used when the API auto-chunks a flat ``list[str]``. - # Matches the voyage-context-4 context window so each string stays a single - # chunk instead of being split. AUTO_CHUNK_SIZE = 32000 def transform_embedding_request( self, model: str, - input: AllEmbeddingInputValues | list[list[str]], - optional_params: dict, - headers: dict, - ) -> dict: + input: AllEmbeddingInputValues | list[list[str]], # mutable-ok: base class signature + optional_params: dict, # mutable-ok: base class signature + headers: dict, # mutable-ok: base class signature + ) -> dict: # mutable-ok: base class signature inputs, extra_params = self._transform_contextual_inputs(input, optional_params) return { "inputs": inputs, @@ -122,9 +122,9 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): @classmethod def _transform_contextual_inputs( cls, - input: AllEmbeddingInputValues | list[list[str]], - optional_params: dict, - ) -> tuple[list[str] | list[list[str]], dict]: + input: AllEmbeddingInputValues | list[list[str]], # mutable-ok: union with AllEmbeddingInputValues + optional_params: dict, # mutable-ok: matches public API + ) -> tuple[list[str] | list[list[str]], dict]: # mutable-ok: returned to caller who owns it """ Normalize ``input`` for Voyage's contextualized embeddings API and return ``(inputs, extra_params)`` where ``extra_params`` carries any @@ -149,32 +149,27 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): Reference: https://docs.voyageai.com/reference/contextualized-embeddings-api """ - # Single string -> a one-element flat list, auto-chunked. if isinstance(input, str): if optional_params.get("input_type") == "query": - return [input], {} - return [input], cls._auto_chunk_params(optional_params) + return [input], {} # mutable-ok: fresh list returned to caller + return [input], cls._auto_chunk_params(optional_params) # mutable-ok: fresh list returned to caller - # Flat list[str]. if isinstance(input, list) and all(isinstance(i, str) for i in input): if optional_params.get("input_type") == "query": - # The API accepts a flat query list as-is. - return input, {} # type: ignore[return-value] - # Otherwise let the API auto-chunk the flat list. - return input, cls._auto_chunk_params(optional_params) # type: ignore[return-value] + return input, {} # pyright: ignore[reportReturnType] # narrowed to list[str] by isinstance+all check + return input, cls._auto_chunk_params(optional_params) # pyright: ignore[reportReturnType] # narrowed to list[str] - # Already list[list[str]] (or another shape) -> pass through unchanged. - return input, {} # type: ignore[return-value] + return input, {} # pyright: ignore[reportReturnType] # list[list[str]] branch @classmethod - def _auto_chunk_params(cls, optional_params: dict) -> dict: + def _auto_chunk_params(cls, optional_params: dict) -> dict: # mutable-ok: matches public API """ Params required to send a flat ``list[str]`` to the contextualized API. ``enable_auto_chunking=True`` requires ``input_type="document"``, so set it unless the caller already provided an ``input_type``. """ - params: dict[str, Any] = { + params: dict[str, Any] = { # mutable-ok: building return value "enable_auto_chunking": True, "chunk_size": cls.AUTO_CHUNK_SIZE, } @@ -189,16 +184,18 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): model_response: EmbeddingResponse, logging_obj: LiteLLMLoggingObj, api_key: str | None = None, - request_data: dict = {}, - optional_params: dict = {}, - litellm_params: dict = {}, + request_data: dict = {}, # mutable-ok: base class signature + optional_params: dict = {}, # mutable-ok: base class signature + litellm_params: dict = {}, # mutable-ok: base class signature ) -> EmbeddingResponse: try: raw_response_json = raw_response.json() except Exception: - raise VoyageError(message=raw_response.text, status_code=raw_response.status_code) + raise VoyageError( + message=raw_response.text, + status_code=raw_response.status_code, + ) - # model_response.usage model_response.model = raw_response_json.get("model") model_response.data = raw_response_json.get("data") model_response.object = raw_response_json.get("object") @@ -210,7 +207,12 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): model_response.usage = usage return model_response - def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict | httpx.Headers, # mutable-ok: base class signature + ) -> BaseLLMException: return VoyageError(message=error_message, status_code=status_code, headers=headers) @staticmethod From af634ab4265d00f3fe730623a7f28d4ddef86d42 Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 29 Jul 2026 22:13:15 +0200 Subject: [PATCH 012/126] test(voyage): add unit tests for contextual embedding in test_litellm (CI coverage path) --- .../test_voyage_contextual_embedding.py | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 tests/test_litellm/llms/voyage/test_voyage_contextual_embedding.py diff --git a/tests/test_litellm/llms/voyage/test_voyage_contextual_embedding.py b/tests/test_litellm/llms/voyage/test_voyage_contextual_embedding.py new file mode 100644 index 00000000000..865608e286f --- /dev/null +++ b/tests/test_litellm/llms/voyage/test_voyage_contextual_embedding.py @@ -0,0 +1,222 @@ +import json +from unittest.mock import MagicMock + +import pytest + + +class TestVoyageContextualEmbeddings: + def test_contextual_model_detection(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + assert VoyageContextualEmbeddingConfig.is_contextualized_embeddings("voyage-context-3") + assert VoyageContextualEmbeddingConfig.is_contextualized_embeddings("voyage-context-4") + assert not VoyageContextualEmbeddingConfig.is_contextualized_embeddings("voyage-3-lite") + + def test_url_generation(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + assert ( + config.get_complete_url(None, None, "voyage-context-4", {}, {}) + == "https://api.voyageai.com/v1/contextualizedembeddings" + ) + assert ( + config.get_complete_url("https://custom.api.com", None, "voyage-context-4", {}, {}) + == "https://custom.api.com/contextualizedembeddings" + ) + assert ( + config.get_complete_url( + "https://custom.api.com/contextualizedembeddings", + None, + "voyage-context-4", + {}, + {}, + ) + == "https://custom.api.com/contextualizedembeddings" + ) + + def test_get_supported_openai_params(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + assert config.get_supported_openai_params("voyage-context-4") == [ + "encoding_format", + "dimensions", + ] + + def test_map_openai_params(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + result = config.map_openai_params( + {"encoding_format": "float", "dimensions": 512}, {}, "voyage-context-4", False + ) + assert result["encoding_format"] == "float" + assert result["output_dimension"] == 512 + + def test_validate_environment_with_api_key(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + headers = config.validate_environment( + {}, "voyage-context-4", [], {}, {}, api_key="test-key" + ) + assert headers == {"Authorization": "Bearer test-key"} + + def test_validate_environment_secret_fallback(self, monkeypatch): + import litellm.llms.voyage.embedding.transformation_contextual as module + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + def fake_get_secret(name): + return "secret-key" if name == "VOYAGE_API_KEY" else None + + monkeypatch.setattr(module, "get_secret_str", fake_get_secret) + config = VoyageContextualEmbeddingConfig() + headers = config.validate_environment( + {}, "voyage-context-4", [], {}, {}, api_key=None + ) + assert headers == {"Authorization": "Bearer secret-key"} + + def test_nested_list_passthrough(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + nested = [["Hello", "world"], ["Test"]] + transformed = config.transform_embedding_request( + "voyage-context-4", nested, {}, {} + ) + assert transformed["inputs"] == nested + assert transformed["model"] == "voyage-context-4" + assert "enable_auto_chunking" not in transformed + + def test_flat_list_str_auto_chunked(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + transformed = config.transform_embedding_request( + "voyage-context-4", ["Hello", "world"], {}, {} + ) + assert transformed["inputs"] == ["Hello", "world"] + assert transformed["enable_auto_chunking"] is True + assert transformed["chunk_size"] == 32000 + assert transformed["input_type"] == "document" + + def test_flat_list_str_query_no_auto_chunk(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + transformed = config.transform_embedding_request( + "voyage-context-4", ["Hello", "world"], {"input_type": "query"}, {} + ) + assert transformed["inputs"] == ["Hello", "world"] + assert transformed["input_type"] == "query" + assert "enable_auto_chunking" not in transformed + + def test_flat_list_str_document_preserves_input_type(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + transformed = config.transform_embedding_request( + "voyage-context-4", ["Hello"], {"input_type": "document"}, {} + ) + assert transformed["input_type"] == "document" + assert transformed["enable_auto_chunking"] is True + + def test_single_string_auto_chunked(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + transformed = config.transform_embedding_request( + "voyage-context-4", "Hello", {}, {} + ) + assert transformed["inputs"] == ["Hello"] + assert transformed["enable_auto_chunking"] is True + assert transformed["input_type"] == "document" + + def test_single_string_query_no_auto_chunk(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + transformed = config.transform_embedding_request( + "voyage-context-4", "Hello", {"input_type": "query"}, {} + ) + assert transformed["inputs"] == ["Hello"] + assert transformed["input_type"] == "query" + assert "enable_auto_chunking" not in transformed + + def test_response_transformation(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + from litellm.types.utils import EmbeddingResponse + + config = VoyageContextualEmbeddingConfig() + response_payload = { + "object": "list", + "data": [{"object": "embedding", "embedding": [0.1, 0.2], "index": 0}], + "model": "voyage-context-4", + "usage": {"total_tokens": 24}, + } + raw_response = MagicMock() + raw_response.json.return_value = response_payload + raw_response.status_code = 200 + raw_response.text = json.dumps(response_payload) + + model_response = EmbeddingResponse() + transformed = config.transform_embedding_response( + "voyage-context-4", raw_response, model_response, MagicMock() + ) + assert transformed.model == "voyage-context-4" + assert transformed.object == "list" + assert transformed.data == response_payload["data"] + assert transformed.usage.prompt_tokens == 24 + assert transformed.usage.total_tokens == 24 + + def test_error_response_and_error_class(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + VoyageError, + ) + from litellm.types.utils import EmbeddingResponse + + config = VoyageContextualEmbeddingConfig() + raw_response = MagicMock() + raw_response.json.side_effect = ValueError("not json") + raw_response.status_code = 400 + raw_response.text = "bad request" + + with pytest.raises(VoyageError) as exc_info: + config.transform_embedding_response( + "voyage-context-4", raw_response, EmbeddingResponse(), MagicMock() + ) + assert exc_info.value.status_code == 400 + assert exc_info.value.message == "bad request" + + error = config.get_error_class("rate limited", 429, {"x-test": "1"}) + assert isinstance(error, VoyageError) + assert error.status_code == 429 + assert error.message == "rate limited" From a1be4236322d7615eece57ed6cc84867e194e866 Mon Sep 17 00:00:00 2001 From: fzowl Date: Sat, 15 Aug 2026 20:29:31 +0200 Subject: [PATCH 013/126] fix(voyage): drop banned typing.Any from contextual transform to satisfy strict lint gate --- litellm/llms/voyage/embedding/transformation_contextual.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index d473450a969..0c02f00fbd5 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -3,7 +3,7 @@ This module is used to transform the request and response for the Voyage context This would be used for all the contextualized embeddings models in Voyage. """ -from typing import Any, Final +from typing import Final import httpx @@ -167,7 +167,7 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): ``enable_auto_chunking=True`` requires ``input_type="document"``, so set it unless the caller already provided an ``input_type``. """ - params: dict[str, Any] = { # mutable-ok: building return value + params: dict[str, object] = { # mutable-ok: building return value "enable_auto_chunking": True, "chunk_size": cls.AUTO_CHUNK_SIZE, } From 8c8cd890adcc14ed8c13d2b315e8eebc6d5c031d Mon Sep 17 00:00:00 2001 From: fzowl Date: Sat, 15 Aug 2026 20:38:40 +0200 Subject: [PATCH 014/126] fix(voyage): drop redundant isinstance list check to satisfy basedpyright budget --- litellm/llms/voyage/embedding/transformation_contextual.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index 0c02f00fbd5..a9a4a01f8e3 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -152,9 +152,9 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): return [input], {} # mutable-ok: fresh list returned to caller return [input], cls._auto_chunk_params(optional_params) # mutable-ok: fresh list returned to caller - if isinstance(input, list) and all(isinstance(i, str) for i in input): + if all(isinstance(i, str) for i in input): if optional_params.get("input_type") == "query": - return input, {} # pyright: ignore[reportReturnType] # narrowed to list[str] by isinstance+all check + return input, {} # pyright: ignore[reportReturnType] # narrowed to list[str] by all(isinstance) check return input, cls._auto_chunk_params(optional_params) # pyright: ignore[reportReturnType] # narrowed to list[str] return input, {} # pyright: ignore[reportReturnType] # list[list[str]] branch From 88b3cfd789b5726af3acb7387dc7acb54dbacd85 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 29 Aug 2026 00:55:43 +0000 Subject: [PATCH 015/126] fix(mcp): bind per-user OAuth credentials to the authenticated LiteLLM caller Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../mcp_server/discoverable_endpoints.py | 20 +- .../mcp_server/mcp_server_manager.py | 2 + .../mcp_server/oauth_identity_binding.py | 249 ++++++++++++++++++ .../mcp_management_endpoints.py | 22 ++ .../types/mcp_server/mcp_server_manager.py | 21 ++ .../mcp_server/test_oauth_identity_binding.py | 240 +++++++++++++++++ .../test_mcp_management_endpoints.py | 61 +++++ 7 files changed, 614 insertions(+), 1 deletion(-) create mode 100644 litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 93b85edd88d..370dee77083 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -54,6 +54,9 @@ from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( relative_request_url, revoke_refresh_token, ) +from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( + enforce_oauth_identity_binding, +) from litellm.proxy._experimental.mcp_server.oauth_utils import ( TOKEN_NO_CACHE_HEADERS, build_upstream_oauth2_token_request, @@ -1133,11 +1136,26 @@ async def exchange_token_with_server( server_id=resolved_server.server_id, ) + # Bind the exchanged token to the LiteLLM caller BEFORE it is returned, stored, or cached, so a + # token minted for a different upstream principal never becomes usable under the caller's user_id. + resolved_user_id: Final = ( + await _extract_user_id_from_request(request) + if resolved_server.needs_user_oauth_token or resolved_server.oauth_identity_binding is not None + else None + ) + if isinstance(token_response, dict): + await enforce_oauth_identity_binding( + server=resolved_server, + token_response=token_response, + litellm_user_id=resolved_user_id, + grant_type=grant_type, + ) + # Store server-side when the server is configured for per-user OAuth and # the calling client has provided a valid LiteLLM identity. # Errors are non-fatal: the token is still returned to the client. if resolved_server.needs_user_oauth_token: - user_id: Final = await _extract_user_id_from_request(request) + user_id: Final = resolved_user_id if user_id: try: await _store_per_user_token_server_side( diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 2330120adad..1d61aba8bc5 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -2174,6 +2174,8 @@ class MCPServerManager: allow_elicitation=bool(server_config.get("allow_elicitation", False)), timeout=server_config.get("timeout", None), max_concurrent_requests=server_config.get("max_concurrent_requests", None), + token_validation=server_config.get("token_validation", None), + oauth_identity_binding=server_config.get("oauth_identity_binding", None), ) self._assign_unique_short_prefix(new_server) _warn_internal_delegate_pkce_if_applicable(new_server, source="config") diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py new file mode 100644 index 00000000000..f132f59ae99 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -0,0 +1,249 @@ +"""Per-user OAuth identity binding: verify the upstream OIDC principal matches the LiteLLM caller. + +Closes the confused-deputy gap where a browser authenticated upstream as one principal produces a +token that the relay stores under a different, LiteLLM-authenticated principal: before the token +endpoint returns, stores, or caches an exchanged token for an identity-bound server, the id_token +is validated (signature via the pinned issuer's JWKS, issuer, audience, expiry) and its principal +claim is compared to the caller's trusted LiteLLM identity. Mismatches fail closed in enforce mode +and are logged in audit mode. +""" + +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from typing import Final, Literal + +import jwt +from fastapi import HTTPException + +from litellm._logging import verbose_logger +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.types.llms.custom_http import httpxSpecialProvider +from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + +_ALLOWED_ID_TOKEN_ALGORITHMS: Final = ( + "RS256", + "RS384", + "RS512", + "ES256", + "ES384", + "ES512", + "PS256", + "PS384", + "PS512", +) +_JWKS_CACHE_TTL_SECONDS: Final = 3600 +_jwks_cache: Final = InMemoryCache(default_ttl=_JWKS_CACHE_TTL_SECONDS) + +JwksFetcher = Callable[[MCPOAuthIdentityBinding], Awaitable[list[Mapping[str, object]]]] +CallerPrincipalLoader = Callable[[str, MCPOAuthIdentityBinding], Awaitable[str | None]] + +_RejectionCode = Literal["oauth_principal_mismatch", "oauth_identity_binding_failed"] + + +@dataclass(frozen=True, slots=True) +class _BindingRejection: + code: _RejectionCode + description: str + + +async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> list[Mapping[str, object]]: + jwks_url: Final[str] = binding.jwks_url or await _discover_jwks_url(binding.issuer) + cached: Final = await _jwks_cache.async_get_cache(jwks_url) + if isinstance(cached, list): + return cached + client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) + response: Final = await client.get(jwks_url) + response.raise_for_status() + document: Final = response.json() + keys: Final = document.get("keys") if isinstance(document, dict) else None + if not isinstance(keys, list): + raise TypeError(f"JWKS document at {jwks_url} has no 'keys' array") + await _jwks_cache.async_set_cache(jwks_url, keys, ttl=_JWKS_CACHE_TTL_SECONDS) + return keys + + +async def _discover_jwks_url(issuer: str) -> str: + discovery_url: Final = f"{issuer.rstrip('/')}/.well-known/openid-configuration" + client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) + response: Final = await client.get(discovery_url) + response.raise_for_status() + metadata: Final = response.json() + jwks_uri: Final = metadata.get("jwks_uri") if isinstance(metadata, dict) else None + if not isinstance(jwks_uri, str) or not jwks_uri: + raise ValueError(f"OIDC discovery at {discovery_url} returned no jwks_uri") + return jwks_uri + + +def _select_signing_key(id_token: str, keys: list[Mapping[str, object]]) -> "jwt.PyJWK | _BindingRejection": + header: Final = jwt.get_unverified_header(id_token) + kid: Final = header.get("kid") + for key in keys: + if kid is None or key.get("kid") == kid: + return jwt.PyJWK(dict(key)) + return _BindingRejection( + code="oauth_identity_binding_failed", + description=f"id_token signing key (kid={kid!r}) not found in the issuer's JWKS", + ) + + +def _decode_id_token( + id_token: str, + binding: MCPOAuthIdentityBinding, + signing_key: "jwt.PyJWK", +) -> "Mapping[str, object] | _BindingRejection": + try: + return jwt.decode( + id_token, + signing_key.key, + algorithms=list(_ALLOWED_ID_TOKEN_ALGORITHMS), + issuer=binding.issuer, + audience=binding.audiences if binding.audiences else None, + options={ + "require": ["iss", "exp"], + "verify_aud": bool(binding.audiences), + }, + ) + except jwt.InvalidTokenError as exc: + return _BindingRejection( + code="oauth_identity_binding_failed", + description=f"id_token validation failed: {exc}", + ) + + +def _upstream_principal( + claims: Mapping[str, object], + binding: MCPOAuthIdentityBinding, +) -> "str | _BindingRejection": + principal: Final = claims.get(binding.principal_claim) + if not isinstance(principal, str) or not principal: + return _BindingRejection( + code="oauth_identity_binding_failed", + description=f"id_token has no usable '{binding.principal_claim}' claim", + ) + if binding.principal_claim == "email" and binding.require_email_verified and claims.get("email_verified") is not True: + return _BindingRejection( + code="oauth_identity_binding_failed", + description="id_token email is not verified (email_verified is not true)", + ) + return principal + + +async def _load_caller_principal(litellm_user_id: str, binding: MCPOAuthIdentityBinding) -> str | None: + if binding.caller_field == "user_id": + return litellm_user_id + from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # inline import avoids a module-load circular import + load_active_user_by_id, + ) + + loaded: Final = await load_active_user_by_id(litellm_user_id) + if isinstance(loaded, str): + return None + return loaded.user_email + + +def _principals_match(upstream: str, caller: str, binding: MCPOAuthIdentityBinding) -> bool: + if binding.principal_claim == "email" or binding.caller_field == "user_email": + return upstream.strip().casefold() == caller.strip().casefold() + return upstream == caller + + +async def _evaluate_binding( + binding: MCPOAuthIdentityBinding, + token_response: Mapping[str, object], + litellm_user_id: str | None, + grant_type: str, + jwks_fetcher: JwksFetcher, + caller_principal_loader: CallerPrincipalLoader, +) -> _BindingRejection | None: + id_token: Final = token_response.get("id_token") + if not isinstance(id_token, str) or not id_token: + if grant_type == "refresh_token": + return None + return _BindingRejection( + code="oauth_identity_binding_failed", + description="the upstream token response carries no id_token to bind the credential to a principal", + ) + if not litellm_user_id: + return _BindingRejection( + code="oauth_identity_binding_failed", + description="the request carries no resolvable LiteLLM user identity to bind the credential to", + ) + try: + keys: Final = await jwks_fetcher(binding) + except Exception as exc: # noqa: BLE001 # a JWKS fetch failure must fail closed, not surface as a 500 + return _BindingRejection( + code="oauth_identity_binding_failed", + description=f"could not fetch the issuer's JWKS: {exc}", + ) + signing_key: Final = _select_signing_key(id_token, keys) + if isinstance(signing_key, _BindingRejection): + return signing_key + claims: Final = _decode_id_token(id_token, binding, signing_key) + if isinstance(claims, _BindingRejection): + return claims + upstream: Final = _upstream_principal(claims, binding) + if isinstance(upstream, _BindingRejection): + return upstream + caller: Final = await caller_principal_loader(litellm_user_id, binding) + if not caller: + return _BindingRejection( + code="oauth_identity_binding_failed", + description=f"the LiteLLM user has no '{binding.caller_field}' to compare the upstream principal against", + ) + if not _principals_match(upstream, caller, binding): + return _BindingRejection( + code="oauth_principal_mismatch", + description="The browser account does not match the selected credential owner.", + ) + return None + + +async def enforce_oauth_identity_binding( + server: MCPServer, + token_response: Mapping[str, object], + litellm_user_id: str | None, + grant_type: str, + jwks_fetcher: JwksFetcher = _fetch_issuer_jwks, + caller_principal_loader: CallerPrincipalLoader = _load_caller_principal, +) -> None: + """Validate the exchanged token's upstream principal against the LiteLLM caller. + + No-op when the server has no binding or it is disabled. In enforce mode a failure raises 403 + before the caller returns, stores, or caches the token; in audit mode failures are logged only. + A refresh_token grant without an id_token is allowed in both modes: the stored credential keeps + the binding established at the original authorization_code exchange. + """ + binding: Final = server.oauth_identity_binding + if binding is None or binding.mode == "disabled": + return + rejection: Final = await _evaluate_binding( + binding=binding, + token_response=token_response, + litellm_user_id=litellm_user_id, + grant_type=grant_type, + jwks_fetcher=jwks_fetcher, + caller_principal_loader=caller_principal_loader, + ) + if rejection is None: + return + if binding.mode == "audit": + verbose_logger.warning( + "oauth_identity_binding audit: server=%s user=%s grant=%s rejected=%s (%s)", + server.server_id, + litellm_user_id, + grant_type, + rejection.code, + rejection.description, + ) + return + raise HTTPException( + status_code=403, + detail={ + "error": rejection.code, + "error_description": rejection.description, + "server_id": server.server_id, + "credential_owner": "caller", + "credential_stored": False, + }, + ) diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 556a30d0b29..5e68e2440da 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -2135,6 +2135,28 @@ if MCP_AVAILABLE: """Persist the OAuth2 access token obtained by the calling user.""" prisma_client: Final = get_prisma_client_or_throw("Database not connected. Connect a database to your proxy") await _authorize_and_fetch_mcp_server(prisma_client, user_api_key_dict, server_id) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 + global_mcp_server_manager as _manager, + ) + + # This endpoint accepts an opaque token with no upstream identity validation, so it must be + # closed for identity-bound servers or it becomes a bypass of the token-relay binding check. + registry_server: Final = _manager.get_mcp_server_by_id(server_id) + binding: Final = registry_server.oauth_identity_binding if registry_server else None + if binding is not None and binding.mode == "enforce": + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={ + "error": "oauth_identity_binding_enforced", + "error_description": ( + "Direct credential storage is disabled for this server: its OAuth identity " + "binding is enforced and this endpoint cannot validate the token's principal. " + "Complete the OAuth flow through the gateway instead." + ), + "server_id": server_id, + "credential_stored": False, + }, + ) user_id: Final = user_api_key_dict.user_id or "" if not user_id: raise HTTPException( diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 9bf3acc601c..cc3f6c8e8d8 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -38,6 +38,26 @@ class MCPOAuthMetadata(BaseModel): usable in memory but must never be persisted as configuration.""" +class MCPOAuthIdentityBinding(BaseModel): + """Per-server policy binding stored per-user OAuth credentials to the authenticated LiteLLM caller. + + When enabled for an interactive oauth2 server, the token relay validates the upstream OIDC + ``id_token`` (signature via the pinned issuer's JWKS, issuer, audience, expiry) and compares its + principal claim to the LiteLLM caller's trusted identity before the token is returned, stored, + or cached. ``audit`` logs mismatches without changing behavior; ``enforce`` fails closed with + 403 ``oauth_principal_mismatch`` and disables the direct ``oauth-user-credential`` POST, which + would otherwise bypass validation with an arbitrary opaque token. + """ + + mode: Literal["disabled", "audit", "enforce"] = "disabled" + issuer: str + jwks_url: str | None = None + audiences: list[str] = [] + principal_claim: str = "email" + caller_field: Literal["user_email", "user_id"] = "user_email" + require_email_verified: bool = True + + class MCPServer(BaseModel): server_id: str name: str @@ -172,6 +192,7 @@ class MCPServer(BaseModel): # response (supports dot-notation for nested fields, e.g. "team.enterprise_id"). # Tokens that fail validation are rejected before storage. token_validation: dict[str, Any] | None = None + oauth_identity_binding: MCPOAuthIdentityBinding | None = None # Optional TTL override (seconds) for the Redis per-user token cache, capped # at the token's expires_in minus the expiry buffer so a cached entry never # outlives the token. Defaults to the token's expires_in minus the expiry diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py new file mode 100644 index 00000000000..69eaabeda4a --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py @@ -0,0 +1,240 @@ +import time +from collections.abc import Mapping +from typing import Final + +import jwt +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from fastapi import HTTPException + +from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( + enforce_oauth_identity_binding, +) +from litellm.types.mcp import MCPTransport +from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + +ISSUER: Final = "https://idp.example.com" +AUDIENCE: Final = "litellm-client" +KID: Final = "test-key" + +_PRIVATE_KEY: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) +_PRIVATE_PEM: Final = _PRIVATE_KEY.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), +) +_PUBLIC_JWK: Final = { + **jwt.algorithms.RSAAlgorithm.to_jwk(_PRIVATE_KEY.public_key(), as_dict=True), + "kid": KID, + "alg": "RS256", + "use": "sig", +} + + +def _sign_id_token(claims: Mapping[str, object]) -> str: + payload: Final = { + "iss": ISSUER, + "aud": AUDIENCE, + "exp": int(time.time()) + 300, + "iat": int(time.time()), + **claims, + } + return jwt.encode(payload, _PRIVATE_PEM, algorithm="RS256", headers={"kid": KID}) + + +async def _jwks_fetcher(_binding: MCPOAuthIdentityBinding) -> list[Mapping[str, object]]: + return [_PUBLIC_JWK] + + +def _caller_loader(email: str | None): + async def load(_user_id: str, _binding: MCPOAuthIdentityBinding) -> str | None: + return email + + return load + + +def _server(mode: str = "enforce", **binding_overrides: object) -> MCPServer: + return MCPServer( + server_id="srv-1", + name="srv-1", + url="https://mcp.example.com", + transport=MCPTransport.http, + oauth_identity_binding=MCPOAuthIdentityBinding( + mode=mode, + issuer=ISSUER, + audiences=[AUDIENCE], + **binding_overrides, + ), + ) + + +@pytest.mark.asyncio +async def test_matching_principal_passes(): + token: Final = _sign_id_token({"email": "Alice@Example.com", "email_verified": True}) + result: Final = await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_mismatched_principal_rejected(): + token: Final = _sign_id_token({"email": "mallory@example.com", "email_verified": True}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.status_code == 403 + assert exc_info.value.detail["error"] == "oauth_principal_mismatch" + assert exc_info.value.detail["credential_stored"] is False + + +@pytest.mark.asyncio +async def test_missing_id_token_rejected_on_authorization_code(): + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id="user-a", + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.status_code == 403 + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +async def test_refresh_without_id_token_allowed(): + result: Final = await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id="user-a", + grant_type="refresh_token", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_refresh_with_mismatched_id_token_rejected(): + token: Final = _sign_id_token({"email": "mallory@example.com", "email_verified": True}) + with pytest.raises(HTTPException): + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="refresh_token", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + + +@pytest.mark.asyncio +async def test_audit_mode_logs_but_does_not_reject(): + token: Final = _sign_id_token({"email": "mallory@example.com", "email_verified": True}) + result: Final = await enforce_oauth_identity_binding( + server=_server(mode="audit"), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_unverified_email_rejected(): + token: Final = _sign_id_token({"email": "alice@example.com", "email_verified": False}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +async def test_wrong_issuer_rejected(): + payload: Final = { + "iss": "https://evil.example.com", + "aud": AUDIENCE, + "exp": int(time.time()) + 300, + "email": "alice@example.com", + "email_verified": True, + } + token: Final = jwt.encode(payload, _PRIVATE_PEM, algorithm="RS256", headers={"kid": KID}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +async def test_no_litellm_identity_rejected(): + token: Final = _sign_id_token({"email": "alice@example.com", "email_verified": True}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id=None, + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_disabled_binding_is_noop(): + result: Final = await enforce_oauth_identity_binding( + server=_server(mode="disabled"), + token_response={"access_token": "at"}, + litellm_user_id=None, + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader(None), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_no_binding_is_noop(): + server: Final = MCPServer( + server_id="srv-2", + name="srv-2", + url="https://mcp.example.com", + transport=MCPTransport.http, + ) + result: Final = await enforce_oauth_identity_binding( + server=server, + token_response={"access_token": "at"}, + litellm_user_id=None, + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader(None), + ) + assert result is None diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index ceb44de5576..1cb36918f11 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -4638,6 +4638,67 @@ async def test_store_mcp_oauth_user_credential_returns_status(): assert result.expires_at == "2099-01-01T00:00:00+00:00" +@pytest.mark.asyncio +async def test_store_mcp_oauth_user_credential_blocked_when_identity_binding_enforced(): + """The direct opaque-token POST must be closed for enforce-mode identity-bound servers, + otherwise it bypasses the token-relay principal check.""" + from litellm.proxy._types import MCPOAuthUserCredentialRequest + from litellm.types.mcp import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + + if not mgmt_endpoints.MCP_AVAILABLE: + pytest.skip("MCP module not installed") + + from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + store_mcp_oauth_user_credential, + ) + + server_id = "srv-binding-1" + bound_server = MCPServer( + server_id=server_id, + name=server_id, + url="https://mcp.example.com", + transport=MCPTransport.http, + oauth_identity_binding=MCPOAuthIdentityBinding(mode="enforce", issuer="https://idp.example.com"), + ) + store_mock = AsyncMock(return_value=None) + + with ( + patch( # test-quality-ok: mirrors the existing store-credential tests in this file + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=_make_prisma_client(), + ), + patch( # test-quality-ok: mirrors the existing store-credential tests in this file + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", + new=AsyncMock(return_value=generate_mock_mcp_server_db_record(server_id=server_id)), + ), + patch( # test-quality-ok: mirrors the existing store-credential tests in this file + "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", + return_value=True, + ), + patch.object( # test-quality-ok: registry is a module-level singleton; injecting it would change the endpoint signature + manager_module.global_mcp_server_manager, + "get_mcp_server_by_id", + return_value=bound_server, + ), + patch( # test-quality-ok: asserting the DB write is never reached is the point of the test + "litellm.proxy.management_endpoints.mcp_management_endpoints.store_user_oauth_credential", + new=store_mock, + ), + ): + with pytest.raises(HTTPException) as exc_info: + await store_mcp_oauth_user_credential( + server_id=server_id, + payload=MCPOAuthUserCredentialRequest(access_token="opaque-tok", expires_in=3600), + user_api_key_dict=_make_user_auth("user-123"), + ) + + assert exc_info.value.status_code == 403 + assert exc_info.value.detail["error"] == "oauth_identity_binding_enforced" + store_mock.assert_not_called() + + @pytest.mark.asyncio async def test_delete_mcp_oauth_user_credential_only_deletes_oauth(): """delete_mcp_oauth_user_credential only deletes OAuth2 credentials, not BYOK.""" From 997a813ab99d6bd4a6b8d52fd9b42e5945c409d4 Mon Sep 17 00:00:00 2001 From: Wolfram Ravenwolf Date: Tue, 1 Sep 2026 23:15:18 +0200 Subject: [PATCH 016/126] fix(wandb): preserve reasoning_effort in chat completions --- litellm/llms/wandb/chat/transformation.py | 3 + .../wandb/test_wandb_chat_transformation.py | 55 ++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/litellm/llms/wandb/chat/transformation.py b/litellm/llms/wandb/chat/transformation.py index a477c00f643..cb9ac7b4c9b 100644 --- a/litellm/llms/wandb/chat/transformation.py +++ b/litellm/llms/wandb/chat/transformation.py @@ -10,6 +10,9 @@ from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig class WandbConfig(OpenAIGPTConfig): + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: inherited contract + return super().get_supported_openai_params(model) + ["reasoning_effort"] # mutable-ok: inherited contract + def map_openai_params( self, non_default_params: dict, diff --git a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py index a5d1eccebe0..4fd66448009 100644 --- a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py +++ b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py @@ -5,7 +5,7 @@ These tests validate the WandbInferenceConfig class which extends OpenAIGPTConfi Nebius AI Studio is an OpenAI-compatible provider with minor customizations. """ - +import json import pytest @@ -17,6 +17,20 @@ from litellm.llms.wandb.chat.transformation import WandbConfig class TestWandbConfig: """Test class for WandB Inference functionality""" + def test_map_openai_params_preserves_reasoning_effort(self): + supported_params = litellm.get_supported_openai_params(model="wandb/openai/gpt-oss-20b") + assert supported_params is not None + assert "reasoning_effort" in supported_params + + result = WandbConfig().map_openai_params( + non_default_params={"reasoning_effort": "medium"}, + optional_params={}, + model="openai/gpt-oss-20b", + drop_params=True, + ) + + assert result == {"reasoning_effort": "medium"} + def test_default_api_base(self): """Test that default API base is used when none is provided""" config = WandbConfig() @@ -139,3 +153,42 @@ class TestWandbConfig: # Check for specific content in the response assert "```python" in content assert "Hey from LiteLLM" in content + + @pytest.mark.respx() + def test_wandb_completion_preserves_reasoning_effort_with_drop_params(self, respx_mock, monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + api_base = "https://api.inference.wandb.ai/v1" + request_mock = respx_mock.post(f"{api_base}/chat/completions").respond( + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-oss-20b", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Done"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + }, + status_code=200, + ) + + completion( + model="wandb/openai/gpt-oss-20b", + messages=[{"role": "user", "content": "Hello"}], + api_key="fake-wandb-key", + api_base=api_base, + reasoning_effort="medium", + drop_params=True, + ) + + request_body = json.loads(request_mock.calls[0].request.content) + assert request_body["reasoning_effort"] == "medium" From 1bf9418917af093d4d3d66885e1c5baa956995cb Mon Sep 17 00:00:00 2001 From: Wolfram Ravenwolf Date: Wed, 2 Sep 2026 02:48:16 +0200 Subject: [PATCH 017/126] fix(wandb): type supported parameter list --- litellm/llms/wandb/chat/transformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/wandb/chat/transformation.py b/litellm/llms/wandb/chat/transformation.py index cb9ac7b4c9b..9c3ff818331 100644 --- a/litellm/llms/wandb/chat/transformation.py +++ b/litellm/llms/wandb/chat/transformation.py @@ -10,7 +10,7 @@ from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig class WandbConfig(OpenAIGPTConfig): - def get_supported_openai_params(self, model: str) -> list: # mutable-ok: inherited contract + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited contract return super().get_supported_openai_params(model) + ["reasoning_effort"] # mutable-ok: inherited contract def map_openai_params( From c60c60e6fe4a860377d9fcd9e611d71234b91e36 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Wed, 2 Sep 2026 10:35:49 +0200 Subject: [PATCH 018/126] fix(proxy): resolve /v1/models limits from the deployment, not the alias --- litellm/proxy/utils.py | 108 +++++++++++---- litellm/router.py | 42 ++++-- litellm/types/router.py | 17 +++ litellm/utils.py | 27 ++-- .../test_model_management_endpoints.py | 4 +- .../proxy/proxy_server/test_routes_models.py | 8 +- .../test_team_model_name_translation.py | 12 +- tests/test_litellm/proxy/test_proxy_utils.py | 127 +++++++++++++++++- tests/test_litellm/test_router.py | 99 ++++++++++++++ 9 files changed, 388 insertions(+), 56 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 051d36c4d0f..b86e6d857cc 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7481,6 +7481,64 @@ async def get_available_models_for_user( return all_models +def _safe_get_model_info(model: str, get_model_info: Callable[[str], ModelInfo]) -> ModelInfo | None: + try: + return get_model_info(model) + except Exception as e: + verbose_proxy_logger.debug( + "create_model_info_response: cost map lookup failed for %s: %s", + model, + e, + ) + return None + + +def _resolve_listing_model_info( + deployment_model: str | None, + listed_model: str, + get_model_info: Callable[[str], ModelInfo], +) -> tuple[ModelInfo, ...]: + """ + Cost-map entries describing a listed model, best source first. + + The name a model is listed under is an arbitrary public alias, so it often misses the + cost map and lands on a fallback-generalization rule that answers with a conservative + family baseline instead of the real model's limits; the deployment's underlying model + is what the request actually reaches. Both names are kept because either can + generalize, and because a deployment's own model is registered into the cost map as a + stub that carries no limits of its own. Exact entries are consulted before generalized + ones, and each field is then taken from the first entry that has it. + """ + listed_info: Final = _safe_get_model_info(listed_model, get_model_info) + + # Fast path, and the only one a wildcard-expanded name takes: with a single name + # there is nothing to order, so skip the generalization test entirely. This keeps + # the per-model cost of the listing on the hot path #33721 exists to protect. + if deployment_model is None or deployment_model == listed_model: + return () if listed_info is None else (listed_info,) + + deployment_info: Final = _safe_get_model_info(deployment_model, get_model_info) + if deployment_info is None: + return () if listed_info is None else (listed_info,) + if listed_info is None: + return (deployment_info,) + + from litellm.utils import is_generalized_model_info + + # Both names resolved: the deployment's model leads unless it only generalized + # while the listed name is an exact cost-map entry. + if is_generalized_model_info(deployment_info) and not is_generalized_model_info(listed_info): + return (listed_info, deployment_info) + return (deployment_info, listed_info) + + +def _first_token_limit(candidates: tuple[ModelInfo, ...], field: str) -> int | None: + return next( + (limit for limit in (coerce_token_limit(info.get(field)) for info in candidates) if limit is not None), + None, + ) + + def create_model_info_response( model_id: str, provider: str, @@ -7505,31 +7563,35 @@ def create_model_info_response( "owned_by": provider, } - try: - model_cost_info: ModelInfo | None = get_model_info(model_id) - except Exception as e: - verbose_proxy_logger.debug( - "create_model_info_response: cost map lookup failed for %s: %s", - model_id, - e, - ) - model_cost_info = None + listing_info: Final = llm_router.get_model_listing_info(model_id) if llm_router is not None else None - max_input_tokens: int | None = None - max_output_tokens: int | None = None - if model_cost_info is not None: - max_input_tokens = coerce_token_limit(model_cost_info.get("max_input_tokens")) - max_output_tokens = coerce_token_limit(model_cost_info.get("max_output_tokens")) - mode: Final = model_cost_info.get("mode") - if isinstance(mode, str): - base["mode"] = mode + candidates: Final = _resolve_listing_model_info( + deployment_model=listing_info.cost_map_key if listing_info is not None else None, + listed_model=model_id, + get_model_info=get_model_info, + ) - if llm_router is not None: - configured_input, configured_output = llm_router.get_configured_token_limits(model_id) - if configured_input is not None: - max_input_tokens = configured_input - if configured_output is not None: - max_output_tokens = configured_output + max_input_tokens: int | None = _first_token_limit(candidates, "max_input_tokens") + max_output_tokens: int | None = _first_token_limit(candidates, "max_output_tokens") + mode: Final = next( + ( + m + for m in ( + cast("Mapping[str, object]", info).get("mode") # cast-ok: an entry need not carry "mode" + for info in candidates + ) + if isinstance(m, str) + ), + None, + ) + if mode is not None: + base["mode"] = mode + + if listing_info is not None: + if listing_info.max_input_tokens is not None: + max_input_tokens = listing_info.max_input_tokens + if listing_info.max_output_tokens is not None: + max_output_tokens = listing_info.max_output_tokens if max_input_tokens is not None: base["max_input_tokens"] = max_input_tokens diff --git a/litellm/router.py b/litellm/router.py index 23d8907fb49..7bc6a6c714a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -200,6 +200,7 @@ from litellm.types.router import ( CredentialLiteLLMParams, CustomRoutingStrategyBase, Deployment, + DeploymentModelListingInfo, DeploymentTypedDict, FallbackAccessCheck, GuardrailTypedDict, @@ -9726,25 +9727,44 @@ class Router: return None return Deployment(**first_usable) if isinstance(first_usable, dict) else first_usable + def get_model_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None: + """ + Return what the concrete deployment behind model_name contributes to its + /v1/models entry: the cost-map key for its underlying model, plus any token + limits explicitly configured in its model_info. Resolved via O(1) index lookup. + + Returns None for wildcard-expanded or unknown names, where the listed name is + the real model name and no deployment-specific information exists, and treats a + malformed configured limit as absent rather than failing the listing. Unlike + get_model_group_info, this never triggers pattern matching or deep copies, so it + is safe to call per listed model on the /v1/models hot path. + """ + deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: + return None + + model_info: Final = deployment.model_info + # base_model is a declared field, so read it as one: an unset or blank value + # means the deployment's own model name is the cost-map key. + base_model: Final = model_info.base_model + return DeploymentModelListingInfo( + cost_map_key=base_model or deployment.litellm_params.model, + max_input_tokens=coerce_token_limit(model_info.get("max_input_tokens")), + max_output_tokens=coerce_token_limit(model_info.get("max_output_tokens")), + ) + def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": """ Return (max_input_tokens, max_output_tokens) explicitly configured in a concrete deployment's model_info for model_name, via O(1) index lookup. Returns (None, None) for wildcard-expanded or unknown names, and treats a - malformed configured value as absent rather than failing the listing. Unlike - get_model_group_info, this never triggers pattern matching or deep copies, so it - is safe to call per listed model on the /v1/models hot path. + malformed configured value as absent rather than failing the caller. """ - deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) - if deployment is None: + listing_info: Final = self.get_model_listing_info(model_name=model_name) + if listing_info is None: return (None, None) - - model_info: Final = deployment.model_info - return ( - coerce_token_limit(model_info.get("max_input_tokens")), - coerce_token_limit(model_info.get("max_output_tokens")), - ) + return (listing_info.max_input_tokens, listing_info.max_output_tokens) def get_deployment_credentials_with_provider( self, model_id: str, team_id: str | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index e0957383aac..6f9fb4bf6ee 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -575,6 +575,23 @@ class Deployment(BaseModel): setattr(self, key, value) +@dataclass(frozen=True, slots=True) +class DeploymentModelListingInfo: + """What a concrete deployment contributes to its OpenAI-compatible listing entry. + + ``cost_map_key`` is the name the deployment's underlying model is known by in + ``litellm.model_cost`` (``model_info.base_model`` when set, else + ``litellm_params.model``), which is what the request actually reaches; the public + model name it is listed under is an arbitrary alias and often absent from the cost + map. The token limits are the ones explicitly set in ``model_info``, which outrank + anything the cost map says. + """ + + cost_map_key: str + max_input_tokens: int | None + max_output_tokens: int | None + + class RouterErrors(enum.Enum): """ Enum for router specific errors with common codes diff --git a/litellm/utils.py b/litellm/utils.py index 252b6756937..80d2399c518 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2948,24 +2948,33 @@ def _resolve_builtin_model_cost_entry(key: str, provider: str) -> dict[str, obje return None +def is_generalized_model_info(model_info: ModelInfo) -> bool: + """Whether ``model_info`` came from a fallback-generalization capability rule. + + Detected as the resolved key missing ``litellm.model_cost`` while matching a + capability rule. A rule-derived entry carries no pricing and only a conservative + family-baseline context window, so callers holding a second candidate name should + prefer an exact cost-map entry from that name over this one. + """ + key: Final = cast("Mapping[str, object]", model_info).get("key") # cast-ok: partial dicts may omit "key" + if not isinstance(key, str): + return False + return key not in litellm.model_cost and match_capability_generalizations(key) is not None + + def _get_builtin_model_info_for_registration(model: str) -> ModelInfo | None: """Resolve ``model`` to its built-in cost-map entry for registration merging. Returns ``None`` when the lookup raises or when it resolved via a - fallback-generalization capability rule, detected as the resolved key missing - ``litellm.model_cost`` while matching a capability rule. A rule-derived entry - carries no pricing, so treating it as a hit would skip the built-in - cache-pricing inheritance for prefix-mangled keys. + fallback-generalization capability rule. A rule-derived entry carries no + pricing, so treating it as a hit would skip the built-in cache-pricing + inheritance for prefix-mangled keys. """ try: info: Final = get_model_info(model=model) except Exception: return None - if info["key"] in litellm.model_cost: - return info - if match_capability_generalizations(info["key"]) is None: - return info - return None + return None if is_generalized_model_info(info) else info _runtime_registered_model_cost: Final[dict[str, dict[str, object]]] = {} # mutable-ok: replayed on reload diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 4661cc17dbc..55e195ff14b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -1944,7 +1944,7 @@ class TestModelInfoEndpoint: ): mock_router.get_fully_blocked_model_names.return_value = set() mock_router.get_model_list.return_value = [] - mock_router.get_configured_token_limits.return_value = (None, None) + mock_router.get_model_listing_info.return_value = None mock_router.get_deployment_by_model_group_name.return_value = Deployment( model_name="gpt-4", litellm_params=LiteLLM_Params(model="openai/gpt-4"), @@ -2021,7 +2021,7 @@ class TestModelInfoEndpoint: ): mock_router.get_fully_blocked_model_names.return_value = set() mock_router.get_model_list.return_value = [] - mock_router.get_configured_token_limits.return_value = (None, None) + mock_router.get_model_listing_info.return_value = None mock_router.get_deployment_by_model_group_name.return_value = Deployment( model_name="team-model-1", litellm_params=LiteLLM_Params(model="custom/team-model-1"), diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_models.py b/tests/test_litellm/proxy/proxy_server/test_routes_models.py index 2b126b1ea95..9063010b366 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_models.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_models.py @@ -17,6 +17,7 @@ import litellm from litellm.proxy import proxy_server from litellm.proxy import utils as proxy_utils from litellm.proxy.utils import create_model_info_response +from litellm.types.router import DeploymentModelListingInfo from .conftest import normalize # type: ignore[import-not-found] @@ -165,13 +166,16 @@ def test_anthropic_format_carries_router_configured_token_limits(client, auth_as built from another entry's lookup shows up as the wrong numbers.""" def _configured(model_name): - return (300000, 32000) if model_name == "gpt-4" else (500000, 4096) + max_input, max_output = (300000, 32000) if model_name == "gpt-4" else (500000, 4096) + return DeploymentModelListingInfo( + cost_map_key=model_name, max_input_tokens=max_input, max_output_tokens=max_output + ) def _cost_map_lookup(model_id): max_input, max_output = (200000, 64000) if model_id == "gpt-4" else (100000, 8000) return {"max_input_tokens": max_input, "max_output_tokens": max_output, "mode": "chat"} - patched_models.get_configured_token_limits = MagicMock(side_effect=_configured) + patched_models.get_model_listing_info = MagicMock(side_effect=_configured) def _resolved(**kwargs): return create_model_info_response(**kwargs, get_model_info=_cost_map_lookup) diff --git a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py index aa35fd64f18..6c1f2eb9bcf 100644 --- a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py @@ -877,7 +877,7 @@ async def test_v1_models_translates_team_model_for_access_group_key(monkeypatch) router.get_model_names.return_value = ["model_name_teamX_uuid9"] router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} router.get_fully_blocked_model_names.return_value = set() - router.get_configured_token_limits.return_value = (None, None) + router.get_model_listing_info.return_value = None router.model_list = [team_dep] router.get_model_list.return_value = [team_dep] @@ -919,7 +919,7 @@ async def test_v1_models_keeps_internal_names_when_public_name_flag_disabled( router.get_model_names.return_value = ["model_name_teamX_uuid9"] router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} router.get_fully_blocked_model_names.return_value = set() - router.get_configured_token_limits.return_value = (None, None) + router.get_model_listing_info.return_value = None router.model_list = [team_dep] router.get_model_list.return_value = [team_dep] @@ -954,7 +954,7 @@ async def test_v1_models_translates_team_model_with_metadata(monkeypatch): router.get_model_names.return_value = ["model_name_teamX_uuid9"] router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} router.get_fully_blocked_model_names.return_value = set() - router.get_configured_token_limits.return_value = (None, None) + router.get_model_listing_info.return_value = None router.model_list = [team_dep] router.get_model_list.return_value = [team_dep] router.get_model_group_info.return_value = None @@ -1000,7 +1000,7 @@ async def test_v1_models_metadata_fallbacks_use_internal_routing_key(monkeypatch router.get_model_names.return_value = ["model_name_teamX_uuid9"] router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} router.get_fully_blocked_model_names.return_value = set() - router.get_configured_token_limits.return_value = (None, None) + router.get_model_listing_info.return_value = None router.model_list = [team_dep] router.get_model_list.return_value = [team_dep] # Fallbacks are keyed on the internal routing name, as the router stores them. @@ -1057,7 +1057,7 @@ async def test_v1_models_metadata_does_not_leak_other_team_fallbacks(monkeypatch router.get_model_names.return_value = ["model_name_teamX_uuid9"] router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} router.get_fully_blocked_model_names.return_value = set() - router.get_configured_token_limits.return_value = (None, None) + router.get_model_listing_info.return_value = None router.model_list = [team_x, team_y] router.get_model_list.return_value = [team_x, team_y] router.fallbacks = [ @@ -1312,7 +1312,7 @@ def test_translate_team_model_names_for_listing_respects_legacy_flag(): def _public_named_router(*team_rows: dict) -> MagicMock: router = MagicMock() router.get_model_list.return_value = list(team_rows) - router.get_configured_token_limits.return_value = (None, None) + router.get_model_listing_info.return_value = None return router diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index dcaad968663..58f62bc4928 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -886,6 +886,7 @@ from typing import cast import litellm from litellm.proxy.utils import create_model_info_response +from litellm.types.router import DeploymentModelListingInfo from litellm.types.utils import ModelInfo @@ -913,7 +914,7 @@ def test_create_model_info_response_includes_max_tokens_from_lookup(): def test_create_model_info_response_does_not_call_router_group_info(): router = MagicMock() - router.get_configured_token_limits.return_value = (None, None) + router.get_model_listing_info.return_value = None response = create_model_info_response( model_id="some-model", @@ -928,7 +929,9 @@ def test_create_model_info_response_does_not_call_router_group_info(): def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map(): router = MagicMock() - router.get_configured_token_limits.return_value = (32000, 8000) + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_key="my-custom-deployment", max_input_tokens=32000, max_output_tokens=8000 + ) response = create_model_info_response( model_id="my-custom-deployment", @@ -944,7 +947,9 @@ def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map( def test_create_model_info_response_deployment_limits_override_cost_map(): router = MagicMock() - router.get_configured_token_limits.return_value = (200000, None) + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_key="gpt-4o", max_input_tokens=200000, max_output_tokens=None + ) response = create_model_info_response( model_id="gpt-4o", @@ -1878,3 +1883,119 @@ async def test_proxy_only_error_5xx_keeps_traceback_and_runs_sync_callbacks(monk Logging.failure_handler = orig_sync_failure assert "test_proxy_utils" in captured["async_traceback"] + + +def test_create_model_info_response_resolves_alias_to_deployment_model(): + """A public model name that is not itself a cost-map key must not be resolved through + the fallback-generalization rules: `bedrock-claude-opus-5` matches the generic + claude-family baseline (200k/64k) by substring, while the deployment it fronts really + accepts 1M/128k. Regression for the /v1/models alias resolution introduced in v1.94.0.""" + from litellm import Router + + saved_model_cost = dict(litellm.model_cost) + try: + router = Router( + model_list=[ + { + "model_name": "bedrock-claude-opus-5", + "litellm_params": { + "custom_llm_provider": "bedrock", + "model": "bedrock/eu.anthropic.claude-opus-5", + }, + "model_info": {"base_model": "eu.anthropic.claude-opus-5"}, + } + ] + ) + + response = create_model_info_response( + model_id="bedrock-claude-opus-5", provider="openai", llm_router=router + ) + finally: + litellm.model_cost.clear() + litellm.model_cost.update(saved_model_cost) + + assert response["max_input_tokens"] == 1000000 + assert response["max_output_tokens"] == 128000 + + +def test_create_model_info_response_keeps_exact_alias_over_generalized_deployment_model(): + """Mirror of the alias bug: when the deployment points at a custom backend name that + only matches a generalization rule, the listed name's exact cost-map entry is the + better answer and must win.""" + from litellm import Router + + saved_model_cost = dict(litellm.model_cost) + try: + router = Router( + model_list=[ + { + "model_name": "claude-opus-5", + "litellm_params": { + "custom_llm_provider": "bedrock", + "model": "bedrock/my-claude-opus-5-provisioned", + }, + } + ] + ) + + response = create_model_info_response( + model_id="claude-opus-5", provider="openai", llm_router=router + ) + finally: + litellm.model_cost.clear() + litellm.model_cost.update(saved_model_cost) + + assert response["max_input_tokens"] == 1000000 + + +def test_create_model_info_response_falls_back_to_alias_for_opaque_deployment_name(): + """An Azure deployment named after the resource rather than the model has no cost-map + entry; the listed name still does, and must keep answering.""" + from litellm import Router + + saved_model_cost = dict(litellm.model_cost) + try: + router = Router( + model_list=[ + { + "model_name": "gpt-4o", + "litellm_params": {"model": "azure/my-gpt4o-deployment"}, + } + ] + ) + + response = create_model_info_response( + model_id="gpt-4o", provider="openai", llm_router=router + ) + finally: + litellm.model_cost.clear() + litellm.model_cost.update(saved_model_cost) + + assert response["max_input_tokens"] == 128000 + assert response["max_output_tokens"] == 16384 + + +def test_create_model_info_response_resolves_mode_through_deployment_model(): + """`mode` is derived from the same lookup, so an aliased embedding deployment + currently reports no mode at all; it must report `embedding`.""" + from litellm import Router + + saved_model_cost = dict(litellm.model_cost) + try: + router = Router( + model_list=[ + { + "model_name": "my-embeddings", + "litellm_params": {"model": "openai/text-embedding-3-small"}, + } + ] + ) + + response = create_model_info_response( + model_id="my-embeddings", provider="openai", llm_router=router + ) + finally: + litellm.model_cost.clear() + litellm.model_cost.update(saved_model_cost) + + assert response["mode"] == "embedding" diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 84f6344be35..37805436f8d 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7271,6 +7271,105 @@ def test_get_configured_token_limits_coerces_numeric_strings(): assert router.get_configured_token_limits("quoted-limits-model") == (32000, 8000) +def test_get_model_listing_info_prefers_base_model_over_litellm_params_model(): + """The cost-map key comes from base_model when set, so a deployment pointing at an + opaque backend name still resolves the real catalog entry.""" + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock-claude-opus-5", + "litellm_params": {"model": "bedrock/eu.anthropic.claude-opus-5"}, + "model_info": {"base_model": "eu.anthropic.claude-opus-5"}, + } + ] + ) + + info = router.get_model_listing_info("bedrock-claude-opus-5") + assert info is not None + assert info.cost_map_key == "eu.anthropic.claude-opus-5" + + +def test_get_model_listing_info_falls_back_to_litellm_params_model(): + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock-claude-opus-5", + "litellm_params": {"model": "bedrock/eu.anthropic.claude-opus-5"}, + } + ] + ) + + info = router.get_model_listing_info("bedrock-claude-opus-5") + assert info is not None + assert info.cost_map_key == "bedrock/eu.anthropic.claude-opus-5" + + +def test_get_model_listing_info_ignores_blank_base_model(): + """A base_model set to an empty string is absent, not a cost-map key.""" + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock-claude-opus-5", + "litellm_params": {"model": "bedrock/eu.anthropic.claude-opus-5"}, + "model_info": {"base_model": ""}, + } + ] + ) + + info = router.get_model_listing_info("bedrock-claude-opus-5") + assert info is not None + assert info.cost_map_key == "bedrock/eu.anthropic.claude-opus-5" + + +def test_get_model_listing_info_returns_none_for_unknown_name(): + router = litellm.Router( + model_list=[ + { + "model_name": "no-limits-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + } + ] + ) + + assert router.get_model_listing_info("not-a-real-model") is None + + +def test_get_model_listing_info_carries_configured_limits(): + router = litellm.Router( + model_list=[ + { + "model_name": "my-custom-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"max_input_tokens": 32000, "max_output_tokens": 8000}, + } + ] + ) + + info = router.get_model_listing_info("my-custom-model") + assert info is not None + assert (info.max_input_tokens, info.max_output_tokens) == (32000, 8000) + + +def test_get_model_listing_info_skips_wildcard_pattern_matching(): + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock/*", + "litellm_params": {"model": "bedrock/*"}, + "model_info": {"max_input_tokens": 12345}, + } + ] + ) + + with patch.object( + router.pattern_router, "route", side_effect=AssertionError("pattern route called") + ): + assert ( + router.get_model_listing_info("bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0") + is None + ) + + @pytest.mark.asyncio async def test_acreate_batch_disable_fallbacks_surfaces_owning_provider_error(): router = litellm.Router( From d5270890c409fe5942ff98c5b3a69370fc0bff81 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Wed, 2 Sep 2026 12:03:36 +0200 Subject: [PATCH 019/126] fix(proxy): report the widest window across a model group, not the first deployment's --- litellm/proxy/utils.py | 45 +++++++-- litellm/router.py | 79 +++++++++++----- litellm/types/router.py | 17 ++-- .../proxy/proxy_server/test_routes_models.py | 2 +- tests/test_litellm/proxy/test_proxy_utils.py | 52 ++++++++++- tests/test_litellm/test_router.py | 91 ++++++++++++++++++- 6 files changed, 242 insertions(+), 44 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index b86e6d857cc..874dca4238f 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7496,10 +7496,11 @@ def _safe_get_model_info(model: str, get_model_info: Callable[[str], ModelInfo]) def _resolve_listing_model_info( deployment_model: str | None, listed_model: str, + listed_info: ModelInfo | None, get_model_info: Callable[[str], ModelInfo], ) -> tuple[ModelInfo, ...]: """ - Cost-map entries describing a listed model, best source first. + Cost-map entries describing one deployment behind a listed model, best source first. The name a model is listed under is an arbitrary public alias, so it often misses the cost map and lands on a fallback-generalization rule that answers with a conservative @@ -7508,9 +7509,10 @@ def _resolve_listing_model_info( generalize, and because a deployment's own model is registered into the cost map as a stub that carries no limits of its own. Exact entries are consulted before generalized ones, and each field is then taken from the first entry that has it. - """ - listed_info: Final = _safe_get_model_info(listed_model, get_model_info) + ``listed_info`` is resolved once by the caller, since a group with several distinct + underlying models resolves the same alias for each of them. + """ # Fast path, and the only one a wildcard-expanded name takes: with a single name # there is nothing to order, so skip the generalization test entirely. This keeps # the per-model cost of the listing on the hot path #33721 exists to protect. @@ -7539,6 +7541,20 @@ def _first_token_limit(candidates: tuple[ModelInfo, ...], field: str) -> int | N ) +def _group_token_limit(candidate_sets: tuple[tuple[ModelInfo, ...], ...], field: str) -> int | None: + """The widest limit any deployment behind the listed name declares for ``field``. + + A model group is normally one model behind several interchangeable deployments, so + there is a single value to report. When a group genuinely mixes models, reporting the + widest window keeps the listing independent of config order and agreeing with + ``/model_group/info``, which aggregates the same way for the Admin UI. + """ + limits: Final = tuple( + limit for limit in (_first_token_limit(candidates, field) for candidates in candidate_sets) if limit is not None + ) + return max(limits) if limits else None + + def create_model_info_response( model_id: str, provider: str, @@ -7565,19 +7581,30 @@ def create_model_info_response( listing_info: Final = llm_router.get_model_listing_info(model_id) if llm_router is not None else None - candidates: Final = _resolve_listing_model_info( - deployment_model=listing_info.cost_map_key if listing_info is not None else None, - listed_model=model_id, - get_model_info=get_model_info, + # One entry per distinct model behind the listed name; (None,) when the router knows + # nothing about it, so the listed name is resolved on its own as before. + deployment_models: Final[tuple[str | None, ...]] = ( + listing_info.cost_map_keys if listing_info is not None and listing_info.cost_map_keys else (None,) + ) + listed_info: Final = _safe_get_model_info(model_id, get_model_info) + candidate_sets: Final = tuple( + _resolve_listing_model_info( + deployment_model=deployment_model, + listed_model=model_id, + listed_info=listed_info, + get_model_info=get_model_info, + ) + for deployment_model in deployment_models ) - max_input_tokens: int | None = _first_token_limit(candidates, "max_input_tokens") - max_output_tokens: int | None = _first_token_limit(candidates, "max_output_tokens") + max_input_tokens: int | None = _group_token_limit(candidate_sets, "max_input_tokens") + max_output_tokens: int | None = _group_token_limit(candidate_sets, "max_output_tokens") mode: Final = next( ( m for m in ( cast("Mapping[str, object]", info).get("mode") # cast-ok: an entry need not carry "mode" + for candidates in candidate_sets for info in candidates ) if isinstance(m, str) diff --git a/litellm/router.py b/litellm/router.py index 7bc6a6c714a..7011f7f504c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9729,29 +9729,57 @@ class Router: def get_model_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None: """ - Return what the concrete deployment behind model_name contributes to its - /v1/models entry: the cost-map key for its underlying model, plus any token - limits explicitly configured in its model_info. Resolved via O(1) index lookup. + Return what the concrete deployments behind model_name contribute to its + /v1/models entry: the cost-map keys for their underlying models, plus the widest + token limits explicitly configured in their model_info. Resolved via O(1) index + lookup. - Returns None for wildcard-expanded or unknown names, where the listed name is - the real model name and no deployment-specific information exists, and treats a - malformed configured limit as absent rather than failing the listing. Unlike - get_model_group_info, this never triggers pattern matching or deep copies, so it - is safe to call per listed model on the /v1/models hot path. + Returns None for wildcard-expanded or unknown names, where the listed name is the + real model name and no deployment-specific information exists, and treats a + malformed configured limit as absent rather than failing the listing. + + The whole group is read rather than just its first deployment, so a group that + mixes models does not advertise a window that depends on config order; the widest + one is reported, which is what get_model_group_info already shows the Admin UI. + Keys are deduplicated, so the ordinary group of interchangeable deployments of one + model still costs the caller a single cost-map lookup. Unlike get_model_group_info, + this never triggers pattern matching or deep copies, so it is safe to call per + listed model on the /v1/models hot path. """ - deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) - if deployment is None: + indices: Final = self.model_name_to_deployment_indices.get(model_name) + if not indices: return None - model_info: Final = deployment.model_info - # base_model is a declared field, so read it as one: an unset or blank value - # means the deployment's own model name is the cost-map key. - base_model: Final = model_info.base_model - return DeploymentModelListingInfo( - cost_map_key=base_model or deployment.litellm_params.model, - max_input_tokens=coerce_token_limit(model_info.get("max_input_tokens")), - max_output_tokens=coerce_token_limit(model_info.get("max_output_tokens")), + deployments: Final = tuple(self.model_list[index] for index in indices) + model_infos: Final = tuple(deployment.get("model_info") or MappingProxyType({}) for deployment in deployments) + params: Final = tuple(deployment.get("litellm_params") or MappingProxyType({}) for deployment in deployments) + # base_model resolution mirrors get_router_model_info: unset or blank means the + # deployment's own model name is the cost-map key. + cost_map_keys: Final = tuple( + dict.fromkeys( # deduplicates while preserving config order + key + for key in ( + model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model") + for model_info, litellm_params in zip(model_infos, params) + ) + if isinstance(key, str) and key + ) ) + return DeploymentModelListingInfo( + cost_map_keys=cost_map_keys, + max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"), + max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"), + ) + + @staticmethod + def _widest_configured_limit(model_infos: Sequence[Mapping[str, Any]], field: str) -> int | None: + """The largest usable value of ``field`` across a group's configured model_info blocks.""" + limits: Final = tuple( + limit + for limit in (coerce_token_limit(model_info.get(field)) for model_info in model_infos) + if limit is not None + ) + return max(limits) if limits else None def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": """ @@ -9760,11 +9788,20 @@ class Router: Returns (None, None) for wildcard-expanded or unknown names, and treats a malformed configured value as absent rather than failing the caller. + + Deliberately reads one deployment rather than aggregating the group the way + get_model_listing_info does: its caller truncates an embedding input to this + value, so the widest window in a mixed group would be the wrong answer there. """ - listing_info: Final = self.get_model_listing_info(model_name=model_name) - if listing_info is None: + deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: return (None, None) - return (listing_info.max_input_tokens, listing_info.max_output_tokens) + + model_info: Final = deployment.model_info + return ( + coerce_token_limit(model_info.get("max_input_tokens")), + coerce_token_limit(model_info.get("max_output_tokens")), + ) def get_deployment_credentials_with_provider( self, model_id: str, team_id: str | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index 6f9fb4bf6ee..a773959f5f1 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -577,17 +577,18 @@ class Deployment(BaseModel): @dataclass(frozen=True, slots=True) class DeploymentModelListingInfo: - """What a concrete deployment contributes to its OpenAI-compatible listing entry. + """What the deployments behind a model name contribute to its OpenAI-compatible listing entry. - ``cost_map_key`` is the name the deployment's underlying model is known by in - ``litellm.model_cost`` (``model_info.base_model`` when set, else - ``litellm_params.model``), which is what the request actually reaches; the public - model name it is listed under is an arbitrary alias and often absent from the cost - map. The token limits are the ones explicitly set in ``model_info``, which outrank - anything the cost map says. + ``cost_map_keys`` are the names those deployments' underlying models are known by in + ``litellm.model_cost`` (``base_model`` when set, else ``litellm_params.model``), which + is what a request actually reaches; the public model name they are listed under is an + arbitrary alias and often absent from the cost map. Keys are deduplicated in config + order, so the ordinary group -- several interchangeable deployments of one model -- + carries exactly one. The token limits are the widest explicitly set in any + deployment's ``model_info``, which outrank anything the cost map says. """ - cost_map_key: str + cost_map_keys: tuple[str, ...] max_input_tokens: int | None max_output_tokens: int | None diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_models.py b/tests/test_litellm/proxy/proxy_server/test_routes_models.py index 9063010b366..54768fef2ce 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_models.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_models.py @@ -168,7 +168,7 @@ def test_anthropic_format_carries_router_configured_token_limits(client, auth_as def _configured(model_name): max_input, max_output = (300000, 32000) if model_name == "gpt-4" else (500000, 4096) return DeploymentModelListingInfo( - cost_map_key=model_name, max_input_tokens=max_input, max_output_tokens=max_output + cost_map_keys=(model_name,), max_input_tokens=max_input, max_output_tokens=max_output ) def _cost_map_lookup(model_id): diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 58f62bc4928..90e7431a22d 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -930,7 +930,7 @@ def test_create_model_info_response_does_not_call_router_group_info(): def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map(): router = MagicMock() router.get_model_listing_info.return_value = DeploymentModelListingInfo( - cost_map_key="my-custom-deployment", max_input_tokens=32000, max_output_tokens=8000 + cost_map_keys=("my-custom-deployment",), max_input_tokens=32000, max_output_tokens=8000 ) response = create_model_info_response( @@ -948,7 +948,7 @@ def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map( def test_create_model_info_response_deployment_limits_override_cost_map(): router = MagicMock() router.get_model_listing_info.return_value = DeploymentModelListingInfo( - cost_map_key="gpt-4o", max_input_tokens=200000, max_output_tokens=None + cost_map_keys=("gpt-4o",), max_input_tokens=200000, max_output_tokens=None ) response = create_model_info_response( @@ -962,6 +962,54 @@ def test_create_model_info_response_deployment_limits_override_cost_map(): assert response["max_output_tokens"] == 16384 +def test_create_model_info_response_reports_widest_window_in_a_mixed_group(): + """A group mixing models advertises the widest window, not whichever is listed first.""" + limits = { + "small-model": _fake_model_info(max_input_tokens=200000, max_output_tokens=4096, mode="chat"), + "large-model": _fake_model_info(max_input_tokens=1000000, max_output_tokens=128000, mode="chat"), + } + + for keys in (("small-model", "large-model"), ("large-model", "small-model")): + router = MagicMock() + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_keys=keys, max_input_tokens=None, max_output_tokens=None + ) + + response = create_model_info_response( + model_id="house-claude", + provider="openai", + llm_router=router, + get_model_info=lambda model: limits[model], + ) + + assert response["max_input_tokens"] == 1000000, keys + assert response["max_output_tokens"] == 128000, keys + + +def test_create_model_info_response_resolves_alias_once_per_listing(): + """The alias is the same for every deployment in the group, so it is looked up once.""" + seen: list[str] = [] + + def _tracking_get_model_info(model: str) -> ModelInfo: + seen.append(model) + return _fake_model_info(max_input_tokens=128000) + + router = MagicMock() + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_keys=("model-a", "model-b"), max_input_tokens=None, max_output_tokens=None + ) + + create_model_info_response( + model_id="house-model", + provider="openai", + llm_router=router, + get_model_info=_tracking_get_model_info, + ) + + assert seen.count("house-model") == 1 + assert sorted(seen) == ["house-model", "model-a", "model-b"] + + def test_create_model_info_response_survives_malformed_configured_limits(): from litellm import Router diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 37805436f8d..588fd98abd1 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7286,7 +7286,7 @@ def test_get_model_listing_info_prefers_base_model_over_litellm_params_model(): info = router.get_model_listing_info("bedrock-claude-opus-5") assert info is not None - assert info.cost_map_key == "eu.anthropic.claude-opus-5" + assert info.cost_map_keys == ("eu.anthropic.claude-opus-5",) def test_get_model_listing_info_falls_back_to_litellm_params_model(): @@ -7301,7 +7301,7 @@ def test_get_model_listing_info_falls_back_to_litellm_params_model(): info = router.get_model_listing_info("bedrock-claude-opus-5") assert info is not None - assert info.cost_map_key == "bedrock/eu.anthropic.claude-opus-5" + assert info.cost_map_keys == ("bedrock/eu.anthropic.claude-opus-5",) def test_get_model_listing_info_ignores_blank_base_model(): @@ -7318,7 +7318,7 @@ def test_get_model_listing_info_ignores_blank_base_model(): info = router.get_model_listing_info("bedrock-claude-opus-5") assert info is not None - assert info.cost_map_key == "bedrock/eu.anthropic.claude-opus-5" + assert info.cost_map_keys == ("bedrock/eu.anthropic.claude-opus-5",) def test_get_model_listing_info_returns_none_for_unknown_name(): @@ -7350,6 +7350,91 @@ def test_get_model_listing_info_carries_configured_limits(): assert (info.max_input_tokens, info.max_output_tokens) == (32000, 8000) +def test_get_model_listing_info_dedupes_interchangeable_deployments(): + """The ordinary group is N deployments of one model, so it yields exactly one key.""" + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-a"}, + }, + { + "model_name": "gpt-4o", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-b"}, + }, + ] + ) + + info = router.get_model_listing_info("gpt-4o") + assert info is not None + assert info.cost_map_keys == ("openai/gpt-4o",) + + +def test_get_model_listing_info_collects_every_model_in_a_mixed_group(): + router = litellm.Router( + model_list=[ + { + "model_name": "house-claude", + "litellm_params": {"model": "anthropic/claude-3-haiku-20240307"}, + }, + { + "model_name": "house-claude", + "litellm_params": {"model": "bedrock/eu.anthropic.claude-opus-5"}, + }, + ] + ) + + info = router.get_model_listing_info("house-claude") + assert info is not None + assert info.cost_map_keys == ( + "anthropic/claude-3-haiku-20240307", + "bedrock/eu.anthropic.claude-opus-5", + ) + + +def test_get_model_listing_info_reports_widest_configured_limits_in_a_mixed_group(): + """Matches how get_model_group_info aggregates for the Admin UI, so the two agree.""" + router = litellm.Router( + model_list=[ + { + "model_name": "house-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"max_input_tokens": 32000, "max_output_tokens": 4096}, + }, + { + "model_name": "house-model", + "litellm_params": {"model": "openai/another-unmapped-model"}, + "model_info": {"max_input_tokens": 128000, "max_output_tokens": 16384}, + }, + ] + ) + + info = router.get_model_listing_info("house-model") + assert info is not None + assert (info.max_input_tokens, info.max_output_tokens) == (128000, 16384) + + +def test_get_model_listing_info_reads_base_model_from_litellm_params(): + """base_model resolution mirrors get_router_model_info, which also accepts it there.""" + router = litellm.Router( + model_list=[ + { + "model_name": "azure-deployment", + "litellm_params": { + "model": "azure/my-azure-deployment-name", + "base_model": "azure/gpt-4o", + "api_key": "sk-a", + "api_base": "https://example.openai.azure.com", + }, + } + ] + ) + + info = router.get_model_listing_info("azure-deployment") + assert info is not None + assert info.cost_map_keys == ("azure/gpt-4o",) + + def test_get_model_listing_info_skips_wildcard_pattern_matching(): router = litellm.Router( model_list=[ From f0f6ff8de4c5bd09b092206003f1f1aa00b7bb41 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Wed, 2 Sep 2026 14:12:23 +0200 Subject: [PATCH 020/126] test(router): cover _widest_configured_limit directly for the coverage gate --- litellm/proxy/utils.py | 14 +++++++++++--- tests/test_litellm/test_router.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 874dca4238f..f891c85320a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7545,9 +7545,17 @@ def _group_token_limit(candidate_sets: tuple[tuple[ModelInfo, ...], ...], field: """The widest limit any deployment behind the listed name declares for ``field``. A model group is normally one model behind several interchangeable deployments, so - there is a single value to report. When a group genuinely mixes models, reporting the - widest window keeps the listing independent of config order and agreeing with - ``/model_group/info``, which aggregates the same way for the Admin UI. + there is a single value to report and the choice of aggregate does not arise. + + When a group genuinely mixes models no single number is right, and the widest is the + deliberate pick over the narrowest for two reasons. It is what ``/model_group/info`` + has long reported to the Admin UI, so the two surfaces agree; disagreeing is the very + complaint this resolution path exists to fix. And of the two ways to be wrong, + under-advertising is worse: a client that trusts a narrowed window silently refuses + prompts the group would have served, while an over-long prompt that reaches a smaller + deployment comes back as a legible context-length error -- and does not reach one at + all when ``enable_pre_call_checks`` is set, which filters deployments the prompt does + not fit. """ limits: Final = tuple( limit for limit in (_first_token_limit(candidates, field) for candidates in candidate_sets) if limit is not None diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 588fd98abd1..eba9a85feb6 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7350,6 +7350,20 @@ def test_get_model_listing_info_carries_configured_limits(): assert (info.max_input_tokens, info.max_output_tokens) == (32000, 8000) +def test_widest_configured_limit_ignores_absent_and_malformed_values(): + model_infos = ( + {"max_input_tokens": 32000}, + {}, + {"max_input_tokens": "not-a-number"}, + {"max_input_tokens": "128000"}, + {"max_output_tokens": 4096}, + ) + + assert litellm.Router._widest_configured_limit(model_infos, "max_input_tokens") == 128000 + assert litellm.Router._widest_configured_limit(model_infos, "max_output_tokens") == 4096 + assert litellm.Router._widest_configured_limit((), "max_input_tokens") is None + + def test_get_model_listing_info_dedupes_interchangeable_deployments(): """The ordinary group is N deployments of one model, so it yields exactly one key.""" router = litellm.Router( From 50874f9fc9cdc9e925d0fc7abda06a178a303980 Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 2 Sep 2026 15:52:47 +0000 Subject: [PATCH 021/126] fix(mcp): require audiences and prove refresh-token ownership for identity binding Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../mcp_server/discoverable_endpoints.py | 9 ++ .../mcp_server/oauth_identity_binding.py | 93 +++++++++++--- .../types/mcp_server/mcp_server_manager.py | 4 +- .../mcp_server/test_oauth_identity_binding.py | 117 +++++++++++++++++- .../test_mcp_management_endpoints.py | 6 +- 5 files changed, 211 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 92128a82856..88f2c47f5fc 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -56,6 +56,8 @@ from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( revoke_refresh_token, ) from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( + RefreshOwnershipProven, + RefreshTokenPresented, enforce_oauth_identity_binding, ) from litellm.proxy._experimental.mcp_server.oauth_utils import ( @@ -1046,7 +1048,13 @@ async def exchange_token_with_server( refresh_request_scope = scope or bridge_upstream_scope if refresh_request_scope: token_data["scope"] = refresh_request_scope + refresh_ownership = ( + RefreshOwnershipProven() + if bridge_upstream_refresh is not None + else RefreshTokenPresented(upstream_refresh_token) + ) else: + refresh_ownership = None if not code: raise HTTPException( status_code=400, @@ -1145,6 +1153,7 @@ async def exchange_token_with_server( token_response=token_response, litellm_user_id=resolved_user_id, grant_type=grant_type, + refresh_ownership=refresh_ownership, ) # Store server-side when the server is configured for per-user OAuth and diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py index f132f59ae99..d4339c15e1a 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -14,6 +14,7 @@ from typing import Final, Literal import jwt from fastapi import HTTPException +from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache @@ -37,6 +38,7 @@ _jwks_cache: Final = InMemoryCache(default_ttl=_JWKS_CACHE_TTL_SECONDS) JwksFetcher = Callable[[MCPOAuthIdentityBinding], Awaitable[list[Mapping[str, object]]]] CallerPrincipalLoader = Callable[[str, MCPOAuthIdentityBinding], Awaitable[str | None]] +StoredRefreshTokenLoader = Callable[[str, str], Awaitable[str | None]] _RejectionCode = Literal["oauth_principal_mismatch", "oauth_identity_binding_failed"] @@ -47,6 +49,19 @@ class _BindingRejection: description: str +@dataclass(frozen=True, slots=True) +class RefreshOwnershipProven: + """The gateway itself unwrapped the upstream refresh token from a sealed per-user envelope.""" + + +@dataclass(frozen=True, slots=True) +class RefreshTokenPresented: + refresh_token: str + + +RefreshOwnership = RefreshOwnershipProven | RefreshTokenPresented | None + + async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> list[Mapping[str, object]]: jwks_url: Final[str] = binding.jwks_url or await _discover_jwks_url(binding.issuer) cached: Final = await _jwks_cache.async_get_cache(jwks_url) @@ -98,11 +113,8 @@ def _decode_id_token( signing_key.key, algorithms=list(_ALLOWED_ID_TOKEN_ALGORITHMS), issuer=binding.issuer, - audience=binding.audiences if binding.audiences else None, - options={ - "require": ["iss", "exp"], - "verify_aud": bool(binding.audiences), - }, + audience=binding.audiences, + options={"require": ["iss", "exp"]}, ) except jwt.InvalidTokenError as exc: return _BindingRejection( @@ -121,7 +133,11 @@ def _upstream_principal( code="oauth_identity_binding_failed", description=f"id_token has no usable '{binding.principal_claim}' claim", ) - if binding.principal_claim == "email" and binding.require_email_verified and claims.get("email_verified") is not True: + if ( + binding.principal_claim == "email" + and binding.require_email_verified + and claims.get("email_verified") is not True + ): return _BindingRejection( code="oauth_identity_binding_failed", description="id_token email is not verified (email_verified is not true)", @@ -142,6 +158,26 @@ async def _load_caller_principal(litellm_user_id: str, binding: MCPOAuthIdentity return loaded.user_email +async def _load_stored_refresh_token(litellm_user_id: str, server_id: str) -> str | None: + try: + from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + get_user_oauth_credential, + ) + from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 + + prisma_client: Final = get_prisma_client_or_throw( + "Database not connected. Cannot verify OAuth refresh token ownership." + ) + cred: Final = await get_user_oauth_credential( + prisma_client=prisma_client, + user_id=litellm_user_id, + server_id=server_id, + ) + return cred.get("refresh_token") if cred else None + except Exception: # noqa: BLE001 # a credential lookup failure must fail closed + return None + + def _principals_match(upstream: str, caller: str, binding: MCPOAuthIdentityBinding) -> bool: if binding.principal_claim == "email" or binding.caller_field == "user_email": return upstream.strip().casefold() == caller.strip().casefold() @@ -153,17 +189,41 @@ async def _evaluate_binding( token_response: Mapping[str, object], litellm_user_id: str | None, grant_type: str, + server_id: str, + refresh_ownership: RefreshOwnership, jwks_fetcher: JwksFetcher, caller_principal_loader: CallerPrincipalLoader, + stored_refresh_token_loader: StoredRefreshTokenLoader, ) -> _BindingRejection | None: id_token: Final = token_response.get("id_token") if not isinstance(id_token, str) or not id_token: - if grant_type == "refresh_token": - return None - return _BindingRejection( - code="oauth_identity_binding_failed", - description="the upstream token response carries no id_token to bind the credential to a principal", - ) + if grant_type != "refresh_token": + return _BindingRejection( + code="oauth_identity_binding_failed", + description="the upstream token response carries no id_token to bind the credential to a principal", + ) + match refresh_ownership: + case RefreshOwnershipProven(): + return None + case None: + return _BindingRejection( + code="oauth_identity_binding_failed", + description="refresh_token grant without an id_token carries no refresh token to prove ownership of", + ) + case RefreshTokenPresented(refresh_token): + if not litellm_user_id: + return _BindingRejection( + code="oauth_identity_binding_failed", + description="the request carries no resolvable LiteLLM user identity to bind the credential to", + ) + stored: Final = await stored_refresh_token_loader(litellm_user_id, server_id) + if stored is None or stored != refresh_token: + return _BindingRejection( + code="oauth_identity_binding_failed", + description="the presented refresh_token is not the caller's stored credential for this server", + ) + return None + assert_never(refresh_ownership) if not litellm_user_id: return _BindingRejection( code="oauth_identity_binding_failed", @@ -204,15 +264,17 @@ async def enforce_oauth_identity_binding( token_response: Mapping[str, object], litellm_user_id: str | None, grant_type: str, + refresh_ownership: RefreshOwnership, jwks_fetcher: JwksFetcher = _fetch_issuer_jwks, caller_principal_loader: CallerPrincipalLoader = _load_caller_principal, + stored_refresh_token_loader: StoredRefreshTokenLoader = _load_stored_refresh_token, ) -> None: """Validate the exchanged token's upstream principal against the LiteLLM caller. No-op when the server has no binding or it is disabled. In enforce mode a failure raises 403 before the caller returns, stores, or caches the token; in audit mode failures are logged only. - A refresh_token grant without an id_token is allowed in both modes: the stored credential keeps - the binding established at the original authorization_code exchange. + A refresh_token grant without an id_token is allowed only when the presented refresh token matches + the caller's stored credential or a sealed bridge envelope already proved ownership. """ binding: Final = server.oauth_identity_binding if binding is None or binding.mode == "disabled": @@ -222,8 +284,11 @@ async def enforce_oauth_identity_binding( token_response=token_response, litellm_user_id=litellm_user_id, grant_type=grant_type, + server_id=server.server_id, + refresh_ownership=refresh_ownership, jwks_fetcher=jwks_fetcher, caller_principal_loader=caller_principal_loader, + stored_refresh_token_loader=stored_refresh_token_loader, ) if rejection is None: return diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index cc3f6c8e8d8..cebc7decac4 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Any, Final, Literal -from pydantic import BaseModel, ConfigDict, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from litellm.types.mcp import ( DEFAULT_SUBJECT_TOKEN_TYPE, @@ -52,7 +52,7 @@ class MCPOAuthIdentityBinding(BaseModel): mode: Literal["disabled", "audit", "enforce"] = "disabled" issuer: str jwks_url: str | None = None - audiences: list[str] = [] + audiences: list[str] = Field(min_length=1) principal_claim: str = "email" caller_field: Literal["user_email", "user_id"] = "user_email" require_email_verified: bool = True diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py index 69eaabeda4a..de59020c44c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py @@ -7,8 +7,11 @@ import pytest from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa from fastapi import HTTPException +from pydantic import ValidationError from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( + RefreshOwnershipProven, + RefreshTokenPresented, enforce_oauth_identity_binding, ) from litellm.types.mcp import MCPTransport @@ -54,6 +57,13 @@ def _caller_loader(email: str | None): return load +def _stored_refresh_token_loader(refresh_token: str | None): + async def load(_user_id: str, _server_id: str) -> str | None: + return refresh_token + + return load + + def _server(mode: str = "enforce", **binding_overrides: object) -> MCPServer: return MCPServer( server_id="srv-1", @@ -77,6 +87,7 @@ async def test_matching_principal_passes(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -92,6 +103,7 @@ async def test_mismatched_principal_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -108,6 +120,7 @@ async def test_missing_id_token_rejected_on_authorization_code(): token_response={"access_token": "at"}, litellm_user_id="user-a", grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -116,18 +129,89 @@ async def test_missing_id_token_rejected_on_authorization_code(): @pytest.mark.asyncio -async def test_refresh_without_id_token_allowed(): +async def test_refresh_without_id_token_allowed_when_presented_token_matches_stored_credential(): result: Final = await enforce_oauth_identity_binding( server=_server(), token_response={"access_token": "at"}, litellm_user_id="user-a", grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented("rt-1"), jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), + stored_refresh_token_loader=_stored_refresh_token_loader("rt-1"), ) assert result is None +@pytest.mark.asyncio +async def test_refresh_without_id_token_rejects_different_presented_token(): + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id="user-a", + grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented("rt-stolen"), + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + stored_refresh_token_loader=_stored_refresh_token_loader("rt-1"), + ) + assert exc_info.value.status_code == 403 + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +async def test_refresh_without_id_token_rejects_missing_stored_token(): + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id="user-a", + grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented("rt-1"), + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + stored_refresh_token_loader=_stored_refresh_token_loader(None), + ) + assert exc_info.value.status_code == 403 + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +async def test_refresh_without_id_token_passes_when_bridge_proves_ownership(): + async def fail_if_called(_user_id: str, _server_id: str) -> str | None: + raise AssertionError("stored refresh token loader should not be called") + + result: Final = await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id="user-a", + grant_type="refresh_token", + refresh_ownership=RefreshOwnershipProven(), + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + stored_refresh_token_loader=fail_if_called, + ) + assert result is None + + +@pytest.mark.asyncio +async def test_refresh_without_id_token_rejects_without_ownership_proof(): + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id="user-a", + grant_type="refresh_token", + refresh_ownership=None, + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + stored_refresh_token_loader=_stored_refresh_token_loader("rt-1"), + ) + assert exc_info.value.status_code == 403 + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + + @pytest.mark.asyncio async def test_refresh_with_mismatched_id_token_rejected(): token: Final = _sign_id_token({"email": "mallory@example.com", "email_verified": True}) @@ -137,6 +221,7 @@ async def test_refresh_with_mismatched_id_token_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="refresh_token", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -150,6 +235,7 @@ async def test_audit_mode_logs_but_does_not_reject(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -165,6 +251,7 @@ async def test_unverified_email_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -187,6 +274,7 @@ async def test_wrong_issuer_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -202,6 +290,7 @@ async def test_no_litellm_identity_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id=None, grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -215,6 +304,7 @@ async def test_disabled_binding_is_noop(): token_response={"access_token": "at"}, litellm_user_id=None, grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader(None), ) @@ -234,7 +324,32 @@ async def test_no_binding_is_noop(): token_response={"access_token": "at"}, litellm_user_id=None, grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader(None), ) assert result is None + + +def test_identity_binding_requires_non_empty_audiences(): + with pytest.raises(ValidationError): + MCPOAuthIdentityBinding(mode="enforce", issuer=ISSUER, audiences=[]) + with pytest.raises(ValidationError): + MCPOAuthIdentityBinding(mode="enforce", issuer=ISSUER) + + +@pytest.mark.asyncio +async def test_wrong_audience_rejected(): + token: Final = _sign_id_token({"aud": "other-client", "email": "alice@example.com", "email_verified": True}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.status_code == 403 + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index bfbbf816e69..f3510e7e0a8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -4674,7 +4674,11 @@ async def test_store_mcp_oauth_user_credential_blocked_when_identity_binding_enf name=server_id, url="https://mcp.example.com", transport=MCPTransport.http, - oauth_identity_binding=MCPOAuthIdentityBinding(mode="enforce", issuer="https://idp.example.com"), + oauth_identity_binding=MCPOAuthIdentityBinding( + mode="enforce", + issuer="https://idp.example.com", + audiences=["litellm-client"], + ), ) store_mock = AsyncMock(return_value=None) From 35db36ee936643d36391eab1cb3c9f5031284b0d Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 2 Sep 2026 16:04:53 +0000 Subject: [PATCH 022/126] fix(mcp): satisfy type discipline lint gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../mcp_server/discoverable_endpoints.py | 4 +- .../mcp_server/oauth_identity_binding.py | 39 ++++++++++++------- .../mcp_management_endpoints.py | 6 +-- .../types/mcp_server/mcp_server_manager.py | 2 +- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 88f2c47f5fc..5f54bbb4958 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -1048,13 +1048,13 @@ async def exchange_token_with_server( refresh_request_scope = scope or bridge_upstream_scope if refresh_request_scope: token_data["scope"] = refresh_request_scope - refresh_ownership = ( + refresh_ownership = ( # rebind-ok: grant-specific branches assign one ownership value RefreshOwnershipProven() if bridge_upstream_refresh is not None else RefreshTokenPresented(upstream_refresh_token) ) else: - refresh_ownership = None + refresh_ownership = None # rebind-ok: grant-specific branches assign one ownership value if not code: raise HTTPException( status_code=400, diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py index d4339c15e1a..b73227f3413 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -8,12 +8,13 @@ claim is compared to the caller's trusted LiteLLM identity. Mismatches fail clos and are logged in audit mode. """ -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass -from typing import Final, Literal +from typing import Final, Literal, TypeAlias import jwt from fastapi import HTTPException +from jwt.types import Options from typing_extensions import assert_never from litellm._logging import verbose_logger @@ -36,11 +37,20 @@ _ALLOWED_ID_TOKEN_ALGORITHMS: Final = ( _JWKS_CACHE_TTL_SECONDS: Final = 3600 _jwks_cache: Final = InMemoryCache(default_ttl=_JWKS_CACHE_TTL_SECONDS) -JwksFetcher = Callable[[MCPOAuthIdentityBinding], Awaitable[list[Mapping[str, object]]]] -CallerPrincipalLoader = Callable[[str, MCPOAuthIdentityBinding], Awaitable[str | None]] -StoredRefreshTokenLoader = Callable[[str, str], Awaitable[str | None]] +JwksFetcher: TypeAlias = Callable[ + [MCPOAuthIdentityBinding], # mutable-ok: Callable parameter syntax requires a list + Awaitable[Sequence[Mapping[str, object]]], +] +CallerPrincipalLoader: TypeAlias = Callable[ + [str, MCPOAuthIdentityBinding], # mutable-ok: Callable parameter syntax requires a list + Awaitable[str | None], +] +StoredRefreshTokenLoader: TypeAlias = Callable[ + [str, str], # mutable-ok: Callable parameter syntax requires a list + Awaitable[str | None], +] -_RejectionCode = Literal["oauth_principal_mismatch", "oauth_identity_binding_failed"] +_RejectionCode: TypeAlias = Literal["oauth_principal_mismatch", "oauth_identity_binding_failed"] @dataclass(frozen=True, slots=True) @@ -59,10 +69,10 @@ class RefreshTokenPresented: refresh_token: str -RefreshOwnership = RefreshOwnershipProven | RefreshTokenPresented | None +RefreshOwnership: TypeAlias = RefreshOwnershipProven | RefreshTokenPresented | None -async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> list[Mapping[str, object]]: +async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> Sequence[Mapping[str, object]]: jwks_url: Final[str] = binding.jwks_url or await _discover_jwks_url(binding.issuer) cached: Final = await _jwks_cache.async_get_cache(jwks_url) if isinstance(cached, list): @@ -90,12 +100,12 @@ async def _discover_jwks_url(issuer: str) -> str: return jwks_uri -def _select_signing_key(id_token: str, keys: list[Mapping[str, object]]) -> "jwt.PyJWK | _BindingRejection": +def _select_signing_key(id_token: str, keys: Sequence[Mapping[str, object]]) -> "jwt.PyJWK | _BindingRejection": header: Final = jwt.get_unverified_header(id_token) kid: Final = header.get("kid") for key in keys: if kid is None or key.get("kid") == kid: - return jwt.PyJWK(dict(key)) + return jwt.PyJWK(dict(key)) # mutable-ok: PyJWT requires a concrete JWK dictionary return _BindingRejection( code="oauth_identity_binding_failed", description=f"id_token signing key (kid={kid!r}) not found in the issuer's JWKS", @@ -108,13 +118,14 @@ def _decode_id_token( signing_key: "jwt.PyJWK", ) -> "Mapping[str, object] | _BindingRejection": try: + decode_options: Final[Options] = {"require": ("iss", "exp")} return jwt.decode( id_token, signing_key.key, - algorithms=list(_ALLOWED_ID_TOKEN_ALGORITHMS), + algorithms=_ALLOWED_ID_TOKEN_ALGORITHMS, issuer=binding.issuer, audience=binding.audiences, - options={"require": ["iss", "exp"]}, + options=decode_options, ) except jwt.InvalidTokenError as exc: return _BindingRejection( @@ -160,10 +171,10 @@ async def _load_caller_principal(litellm_user_id: str, binding: MCPOAuthIdentity async def _load_stored_refresh_token(litellm_user_id: str, server_id: str) -> str | None: try: - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 # keep database imports lazy get_user_oauth_credential, ) - from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 + from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 # keep database imports lazy prisma_client: Final = get_prisma_client_or_throw( "Database not connected. Cannot verify OAuth refresh token ownership." diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 92a6997abff..109030a2df5 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -259,7 +259,7 @@ if MCP_AVAILABLE: if normalize_upstream_header_name(raw) is None: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail={ + detail={ # mutable-ok: FastAPI exception detail requires a JSON-serializable dictionary "error": ( f"Invalid upstream_token_header {raw!r}: must be a valid HTTP header name " "(RFC 7230 token, e.g. 'esb-oauth')" @@ -2256,7 +2256,7 @@ if MCP_AVAILABLE: """Persist the OAuth2 access token obtained by the calling user.""" prisma_client: Final = get_prisma_client_or_throw("Database not connected. Connect a database to your proxy") await _authorize_and_fetch_mcp_server(prisma_client, user_api_key_dict, server_id) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # keep manager import lazy global_mcp_server_manager as _manager, ) @@ -2267,7 +2267,7 @@ if MCP_AVAILABLE: if binding is not None and binding.mode == "enforce": raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, - detail={ + detail={ # mutable-ok: FastAPI exception detail requires a JSON-serializable dictionary "error": "oauth_identity_binding_enforced", "error_description": ( "Direct credential storage is disabled for this server: its OAuth identity " diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index cebc7decac4..c38f1c366b1 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -52,7 +52,7 @@ class MCPOAuthIdentityBinding(BaseModel): mode: Literal["disabled", "audit", "enforce"] = "disabled" issuer: str jwks_url: str | None = None - audiences: list[str] = Field(min_length=1) + audiences: list[str] = Field(min_length=1) # mutable-ok: public Pydantic schema requires list values principal_claim: str = "email" caller_field: Literal["user_email", "user_id"] = "user_email" require_email_verified: bool = True From 22e3fcba54c2409a46e04ba81bd6885cbfc0442a Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 2 Sep 2026 16:05:37 +0000 Subject: [PATCH 023/126] fix(mcp): restore unrelated exception formatting Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/management_endpoints/mcp_management_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 109030a2df5..fd6294fd77e 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -259,7 +259,7 @@ if MCP_AVAILABLE: if normalize_upstream_header_name(raw) is None: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail={ # mutable-ok: FastAPI exception detail requires a JSON-serializable dictionary + detail={ "error": ( f"Invalid upstream_token_header {raw!r}: must be a valid HTTP header name " "(RFC 7230 token, e.g. 'esb-oauth')" From 7ca9f28575021d026d28bb0bfb11a667ba3add77 Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 2 Sep 2026 16:33:59 +0000 Subject: [PATCH 024/126] test(mcp): cover OAuth identity binding paths Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../mcp_server/oauth_identity_binding.py | 2 +- .../mcp_server/test_discoverable_endpoints.py | 126 ++++++++++ .../mcp_server/test_oauth_identity_binding.py | 229 ++++++++++++++++++ 3 files changed, 356 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py index b73227f3413..58ec3cfdb4f 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -234,7 +234,7 @@ async def _evaluate_binding( description="the presented refresh_token is not the caller's stored credential for this server", ) return None - assert_never(refresh_ownership) + assert_never(refresh_ownership) # pragma: no cover if not litellm_user_id: return _BindingRejection( code="oauth_identity_binding_failed", diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 598e9276423..4ae1b105dbf 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -7816,6 +7816,132 @@ async def test_token_exchange_pairs_client_secret_with_server_client_id(): assert "client_secret" not in sent +@pytest.mark.asyncio +async def test_token_exchange_refresh_passes_presented_refresh_ownership(): + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + exchange_token_with_server, + ) + from litellm.proxy._experimental.mcp_server.oauth_identity_binding import RefreshTokenPresented + from litellm.proxy._types import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + + server = MCPServer( + server_id="srv-1", + name="srv-1", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="cid", + token_url="https://provider.example/token", + oauth_identity_binding=MCPOAuthIdentityBinding( + mode="enforce", + issuer="https://provider.example", + audiences=["cid"], + ), + ) + request = MagicMock(spec=Request) + request.base_url = "https://litellm.example.com/" + request.headers = {} + response = MagicMock() + response.json.return_value = {"access_token": "at"} + response.raise_for_status = MagicMock() + client = MagicMock() + client.post = AsyncMock(return_value=response) + enforce = AsyncMock() + + with ( + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + return_value=client, + ), + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._extract_user_id_from_request", + new=AsyncMock(return_value="user-a"), + ), + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.enforce_oauth_identity_binding", + new=enforce, + ), + ): + await exchange_token_with_server( + request=request, + mcp_server=server, + grant_type="refresh_token", + code=None, + redirect_uri=None, + client_id="cid", + client_secret=None, + code_verifier=None, + refresh_token="rt-1", + ) + + ownership = enforce.await_args.kwargs["refresh_ownership"] + assert isinstance(ownership, RefreshTokenPresented) + assert ownership.refresh_token == "rt-1" + + +@pytest.mark.asyncio +async def test_token_exchange_authorization_code_passes_no_refresh_ownership(): + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + exchange_token_with_server, + ) + from litellm.proxy._types import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + + server = MCPServer( + server_id="srv-1", + name="srv-1", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="cid", + token_url="https://provider.example/token", + oauth_identity_binding=MCPOAuthIdentityBinding( + mode="enforce", + issuer="https://provider.example", + audiences=["cid"], + ), + ) + request = MagicMock(spec=Request) + request.base_url = "https://litellm.example.com/" + request.headers = {} + response = MagicMock() + response.json.return_value = {"access_token": "at"} + response.raise_for_status = MagicMock() + client = MagicMock() + client.post = AsyncMock(return_value=response) + enforce = AsyncMock() + + with ( + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + return_value=client, + ), + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._extract_user_id_from_request", + new=AsyncMock(return_value="user-a"), + ), + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.enforce_oauth_identity_binding", + new=enforce, + ), + ): + await exchange_token_with_server( + request=request, + mcp_server=server, + grant_type="authorization_code", + code="auth-code", + redirect_uri="https://litellm.example.com/callback", + client_id="cid", + client_secret=None, + code_verifier=None, + ) + + assert enforce.await_args.kwargs["refresh_ownership"] is None + + def _upstream_token_response(status_code: int, *, json_body: object = None, text_body: str = "") -> "httpx.Response": import httpx diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py index de59020c44c..c2662f9f298 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py @@ -1,6 +1,8 @@ import time from collections.abc import Mapping +from types import SimpleNamespace from typing import Final +from unittest.mock import AsyncMock, MagicMock, patch import jwt import pytest @@ -12,6 +14,11 @@ from pydantic import ValidationError from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( RefreshOwnershipProven, RefreshTokenPresented, + _discover_jwks_url, + _fetch_issuer_jwks, + _load_caller_principal, + _load_stored_refresh_token, + _select_signing_key, enforce_oauth_identity_binding, ) from litellm.types.mcp import MCPTransport @@ -79,6 +86,121 @@ def _server(mode: str = "enforce", **binding_overrides: object) -> MCPServer: ) +@pytest.mark.asyncio +async def test_fetch_issuer_jwks_fetches_and_caches_keys(): + binding: Final = _server(jwks_url="https://idp.example.com/jwks-cache").oauth_identity_binding + assert binding is not None + response: Final = MagicMock() + response.json.return_value = {"keys": [_PUBLIC_JWK]} + client: Final = MagicMock() + client.get = AsyncMock(return_value=response) + + with patch( + "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", + return_value=client, + ): + first: Final = await _fetch_issuer_jwks(binding) + second: Final = await _fetch_issuer_jwks(binding) + + assert first == second == [_PUBLIC_JWK] + client.get.assert_awaited_once_with("https://idp.example.com/jwks-cache") + + +@pytest.mark.asyncio +async def test_fetch_issuer_jwks_rejects_malformed_document(): + binding: Final = _server(jwks_url="https://idp.example.com/jwks-invalid").oauth_identity_binding + assert binding is not None + response: Final = MagicMock() + response.json.return_value = {} + client: Final = MagicMock() + client.get = AsyncMock(return_value=response) + + with patch( + "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", + return_value=client, + ): + with pytest.raises(TypeError, match="has no 'keys' array"): + await _fetch_issuer_jwks(binding) + + +@pytest.mark.asyncio +async def test_discover_jwks_url_returns_provider_uri(): + response: Final = MagicMock() + response.json.return_value = {"jwks_uri": "https://idp.example.com/jwks"} + client: Final = MagicMock() + client.get = AsyncMock(return_value=response) + + with patch( + "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", + return_value=client, + ): + result: Final = await _discover_jwks_url("https://idp.example.com/") + + assert result == "https://idp.example.com/jwks" + client.get.assert_awaited_once_with("https://idp.example.com/.well-known/openid-configuration") + + +@pytest.mark.asyncio +async def test_discover_jwks_url_rejects_missing_provider_uri(): + response: Final = MagicMock() + response.json.return_value = {} + client: Final = MagicMock() + client.get = AsyncMock(return_value=response) + + with patch( + "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", + return_value=client, + ): + with pytest.raises(ValueError, match="returned no jwks_uri"): + await _discover_jwks_url("https://idp.example.com") + + +def test_select_signing_key_returns_matching_key_or_rejection(): + token: Final = _sign_id_token({"email": "alice@example.com", "email_verified": True}) + + selected: Final = _select_signing_key(token, [_PUBLIC_JWK]) + rejected: Final = _select_signing_key(token, []) + + assert selected.__class__.__name__ == "PyJWK" + assert rejected.code == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +async def test_load_caller_principal_supports_user_id_and_database_email(): + user_id_binding: Final = _server(caller_field="user_id").oauth_identity_binding + assert user_id_binding is not None + assert await _load_caller_principal("user-a", user_id_binding) == "user-a" + + with patch( + "litellm.proxy._experimental.mcp_server.bridge_token_flow.load_active_user_by_id", + new=AsyncMock(side_effect=["no_active_key", SimpleNamespace(user_email="alice@example.com")]), + ): + assert await _load_caller_principal("user-a", _server().oauth_identity_binding) is None + assert await _load_caller_principal("user-a", _server().oauth_identity_binding) == "alice@example.com" + + +@pytest.mark.asyncio +async def test_load_stored_refresh_token_returns_credential_and_fails_closed(): + get_credential: Final = AsyncMock(return_value={"refresh_token": "rt-1"}) + with ( + patch( + "litellm.proxy._experimental.mcp_server.db.get_user_oauth_credential", + new=get_credential, + ), + patch( + "litellm.proxy.utils.get_prisma_client_or_throw", + return_value="prisma", + ), + ): + assert await _load_stored_refresh_token("user-a", "srv-1") == "rt-1" + + with patch( + "litellm.proxy.utils.get_prisma_client_or_throw", + side_effect=RuntimeError("database unavailable"), + ): + assert await _load_stored_refresh_token("user-a", "srv-1") is None + + @pytest.mark.asyncio async def test_matching_principal_passes(): token: Final = _sign_id_token({"email": "Alice@Example.com", "email_verified": True}) @@ -112,6 +234,113 @@ async def test_mismatched_principal_rejected(): assert exc_info.value.detail["credential_stored"] is False +@pytest.mark.asyncio +async def test_missing_upstream_principal_rejected(): + token: Final = _sign_id_token({"email_verified": True}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + assert "no usable" in exc_info.value.detail["error_description"] + + +@pytest.mark.asyncio +async def test_jwks_fetch_failure_is_rejected(): + async def fail(_binding: MCPOAuthIdentityBinding) -> list[Mapping[str, object]]: + raise RuntimeError("jwks unavailable") + + token: Final = _sign_id_token({"email": "alice@example.com", "email_verified": True}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + jwks_fetcher=fail, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + assert "jwks unavailable" in exc_info.value.detail["error_description"] + + +@pytest.mark.asyncio +async def test_missing_signing_key_is_rejected(): + token: Final = _sign_id_token({"email": "alice@example.com", "email_verified": True}) + + async def no_keys(_binding: MCPOAuthIdentityBinding) -> tuple[Mapping[str, object], ...]: + return () + + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + jwks_fetcher=no_keys, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + assert "signing key" in exc_info.value.detail["error_description"] + + +@pytest.mark.asyncio +async def test_missing_caller_principal_is_rejected(): + token: Final = _sign_id_token({"email": "alice@example.com", "email_verified": True}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader(None), + ) + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + assert "has no" in exc_info.value.detail["error_description"] + + +@pytest.mark.asyncio +async def test_refresh_without_id_token_requires_litellm_identity_for_presented_token(): + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id=None, + grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented("rt-1"), + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + stored_refresh_token_loader=_stored_refresh_token_loader("rt-1"), + ) + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + assert "no resolvable LiteLLM user identity" in exc_info.value.detail["error_description"] + + +@pytest.mark.asyncio +async def test_user_id_principal_matching_uses_exact_comparison(): + token: Final = _sign_id_token({"sub": "user-a"}) + result: Final = await enforce_oauth_identity_binding( + server=_server(principal_claim="sub", caller_field="user_id"), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("user-a"), + ) + assert result is None + + @pytest.mark.asyncio async def test_missing_id_token_rejected_on_authorization_code(): with pytest.raises(HTTPException) as exc_info: From 540e528d2190e6ed7292174e1985673a602a3018 Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 2 Sep 2026 16:43:18 +0000 Subject: [PATCH 025/126] test(mcp): satisfy test quality gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../mcp_server/test_discoverable_endpoints.py | 16 +++++++++------- .../mcp_server/test_oauth_identity_binding.py | 16 ++++++++-------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 4ae1b105dbf..33b71f8d50f 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -7851,15 +7851,15 @@ async def test_token_exchange_refresh_passes_presented_refresh_ownership(): enforce = AsyncMock() with ( - patch( + patch( # test-quality-ok: no injection seam exists for the exchange's HTTP and identity collaborators "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=client, ), - patch( + patch( # test-quality-ok: no injection seam exists for request identity extraction "litellm.proxy._experimental.mcp_server.discoverable_endpoints._extract_user_id_from_request", new=AsyncMock(return_value="user-a"), ), - patch( + patch( # test-quality-ok: captures the ownership value at the exchange boundary "litellm.proxy._experimental.mcp_server.discoverable_endpoints.enforce_oauth_identity_binding", new=enforce, ), @@ -7915,20 +7915,20 @@ async def test_token_exchange_authorization_code_passes_no_refresh_ownership(): enforce = AsyncMock() with ( - patch( + patch( # test-quality-ok: no injection seam exists for the exchange's HTTP and identity collaborators "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=client, ), - patch( + patch( # test-quality-ok: no injection seam exists for request identity extraction "litellm.proxy._experimental.mcp_server.discoverable_endpoints._extract_user_id_from_request", new=AsyncMock(return_value="user-a"), ), - patch( + patch( # test-quality-ok: captures the ownership value at the exchange boundary "litellm.proxy._experimental.mcp_server.discoverable_endpoints.enforce_oauth_identity_binding", new=enforce, ), ): - await exchange_token_with_server( + result = await exchange_token_with_server( request=request, mcp_server=server, grant_type="authorization_code", @@ -7939,6 +7939,8 @@ async def test_token_exchange_authorization_code_passes_no_refresh_ownership(): code_verifier=None, ) + assert result.status_code == 200 + assert json.loads(result.body)["access_token"] == "at" assert enforce.await_args.kwargs["refresh_ownership"] is None diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py index c2662f9f298..36364c32ae5 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py @@ -95,7 +95,7 @@ async def test_fetch_issuer_jwks_fetches_and_caches_keys(): client: Final = MagicMock() client.get = AsyncMock(return_value=response) - with patch( + with patch( # test-quality-ok: no HTTP dependency injection seam exists for JWKS fetching "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", return_value=client, ): @@ -115,7 +115,7 @@ async def test_fetch_issuer_jwks_rejects_malformed_document(): client: Final = MagicMock() client.get = AsyncMock(return_value=response) - with patch( + with patch( # test-quality-ok: no HTTP dependency injection seam exists for JWKS fetching "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", return_value=client, ): @@ -130,7 +130,7 @@ async def test_discover_jwks_url_returns_provider_uri(): client: Final = MagicMock() client.get = AsyncMock(return_value=response) - with patch( + with patch( # test-quality-ok: no HTTP dependency injection seam exists for OIDC discovery "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", return_value=client, ): @@ -147,7 +147,7 @@ async def test_discover_jwks_url_rejects_missing_provider_uri(): client: Final = MagicMock() client.get = AsyncMock(return_value=response) - with patch( + with patch( # test-quality-ok: no HTTP dependency injection seam exists for OIDC discovery "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", return_value=client, ): @@ -171,7 +171,7 @@ async def test_load_caller_principal_supports_user_id_and_database_email(): assert user_id_binding is not None assert await _load_caller_principal("user-a", user_id_binding) == "user-a" - with patch( + with patch( # test-quality-ok: caller loading is a lazy database boundary without injection "litellm.proxy._experimental.mcp_server.bridge_token_flow.load_active_user_by_id", new=AsyncMock(side_effect=["no_active_key", SimpleNamespace(user_email="alice@example.com")]), ): @@ -183,18 +183,18 @@ async def test_load_caller_principal_supports_user_id_and_database_email(): async def test_load_stored_refresh_token_returns_credential_and_fails_closed(): get_credential: Final = AsyncMock(return_value={"refresh_token": "rt-1"}) with ( - patch( + patch( # test-quality-ok: stored-token loading is a lazy database boundary without injection "litellm.proxy._experimental.mcp_server.db.get_user_oauth_credential", new=get_credential, ), - patch( + patch( # test-quality-ok: stored-token loading is a lazy database boundary without injection "litellm.proxy.utils.get_prisma_client_or_throw", return_value="prisma", ), ): assert await _load_stored_refresh_token("user-a", "srv-1") == "rt-1" - with patch( + with patch( # test-quality-ok: stored-token loading is a lazy database boundary without injection "litellm.proxy.utils.get_prisma_client_or_throw", side_effect=RuntimeError("database unavailable"), ): From eddb29d90ed45849967c2318f15c2edd3e991d8f Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Thu, 3 Sep 2026 11:33:44 +0200 Subject: [PATCH 026/126] fix(proxy): dedup latest health checks in SQL and gate the DB save per window --- litellm/constants.py | 1 + litellm/proxy/db/health_check_latest.py | 96 +++++++++ litellm/proxy/proxy_server.py | 58 +++++- litellm/proxy/utils.py | 52 +---- .../proxy/db/test_health_check_latest.py | 111 ++++++++++ .../proxy_server/test_background_health.py | 81 ++++++++ .../proxy/test_health_check_functions.py | 190 ++++++++++-------- .../test_prisma_client_health.py | 74 ++++--- 8 files changed, 502 insertions(+), 161 deletions(-) create mode 100644 litellm/proxy/db/health_check_latest.py create mode 100644 tests/test_litellm/proxy/db/test_health_check_latest.py diff --git a/litellm/constants.py b/litellm/constants.py index ef9329b9dfc..1f5bd673997 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1619,6 +1619,7 @@ CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME: Final = "cloudzero_export_usage_data" MAVVRIK_FOCUS_EXPORT_JOB_NAME: Final = "mavvrik_focus_export_usage_data" CLOUDZERO_MAX_FETCHED_DATA_RECORDS: Final = int(os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)) SPEND_LOG_CLEANUP_JOB_NAME: Final = "spend_log_cleanup" +BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME: Final = "background_health_check_db_save" KEY_ROTATION_JOB_NAME: Final = "litellm_key_rotation_job" EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME: Final = "litellm_expired_ui_session_key_cleanup_job" WEEKLY_SPEND_REPORT_JOB_ID: Final = "weekly_spend_report_job" diff --git a/litellm/proxy/db/health_check_latest.py b/litellm/proxy/db/health_check_latest.py new file mode 100644 index 00000000000..35bc838379c --- /dev/null +++ b/litellm/proxy/db/health_check_latest.py @@ -0,0 +1,96 @@ +""" +Latest health-check row per model, deduplicated by Postgres. + +prisma-client-py's ``find_many(distinct=...)`` dedups client-side: the emitted +SQL carries no DISTINCT, so the whole append-only history table streams to the +worker on every call. ``SELECT DISTINCT ON`` keeps the transfer at one row per +(model_id, model_name) and is served by the matching descending index. +""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Final + +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, field_validator + +from litellm._logging import verbose_proxy_logger + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + +LATEST_HEALTH_CHECKS_SQL: Final = """ +SELECT DISTINCT ON ("model_id", "model_name") + "health_check_id", "model_name", "model_id", "status", + "healthy_count", "unhealthy_count", "error_message", + "response_time_ms", "details", "checked_by", + "checked_at", "created_at", "updated_at" +FROM "LiteLLM_HealthCheckTable" +ORDER BY "model_id" ASC, "model_name" ASC, "checked_at" DESC +""" + +LATEST_HEALTH_CHECKS_FOR_MODELS_SQL: Final = """ +SELECT DISTINCT ON ("model_id", "model_name") + "health_check_id", "model_name", "model_id", "status", + "healthy_count", "unhealthy_count", "error_message", + "response_time_ms", "details", "checked_by", + "checked_at", "created_at", "updated_at" +FROM "LiteLLM_HealthCheckTable" +WHERE "model_name" = ANY($1) +ORDER BY "model_id" ASC, "model_name" ASC, "checked_at" DESC +""" + + +class LatestHealthCheckRow(BaseModel): + model_config = ConfigDict(frozen=True, protected_namespaces=()) + + health_check_id: str + model_name: str + model_id: str | None = None + status: str + healthy_count: int = 0 + unhealthy_count: int = 0 + error_message: str | None = None + response_time_ms: float | None = None + details: JsonValue | None = None + checked_by: str | None = None + checked_at: datetime + created_at: datetime + updated_at: datetime + + @field_validator("details", mode="before") + @classmethod + def _decode_json_text(cls, value: object) -> object: + return json.loads(value) if isinstance(value, str) else value + + @field_validator("checked_at", "created_at", "updated_at") + @classmethod + def _assume_utc(cls, value: datetime) -> datetime: + return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value + + +_ROWS_ADAPTER: Final = TypeAdapter(tuple[LatestHealthCheckRow, ...]) + + +async def fetch_latest_health_checks(prisma_client: PrismaClient) -> tuple[LatestHealthCheckRow, ...]: + try: + rows: Final = await prisma_client.db.query_raw(LATEST_HEALTH_CHECKS_SQL) + return _ROWS_ADAPTER.validate_python(rows) + except Exception as query_err: # noqa: BLE001 # health decorates other reads; a driver error must not fail them + verbose_proxy_logger.error("Error getting all latest health checks: %s", query_err) + return () + + +async def fetch_latest_health_checks_for_models( + prisma_client: PrismaClient, model_names: Sequence[str] +) -> tuple[LatestHealthCheckRow, ...]: + if not model_names: + return () + try: + rows: Final = await prisma_client.db.query_raw(LATEST_HEALTH_CHECKS_FOR_MODELS_SQL, list(model_names)) + return _ROWS_ADAPTER.validate_python(rows) + except Exception as query_err: # noqa: BLE001 # a paged model list must not fail on its health decoration + verbose_proxy_logger.error("Error getting latest health checks for models: %s", query_err) + return () diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 27132c90e05..0e330272bdf 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16,7 +16,16 @@ import threading import time import traceback import warnings -from collections.abc import AsyncGenerator, AsyncIterator, Callable, Collection, Mapping, MutableMapping, Sequence +from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Awaitable, + Callable, + Collection, + Mapping, + MutableMapping, + Sequence, +) from datetime import datetime, timedelta, timezone from types import MappingProxyType, UnionType from typing import ( @@ -50,6 +59,7 @@ from litellm.constants import ( AIOHTTP_NEEDS_CLEANUP_CLOSED, AIOHTTP_TTL_DNS_CACHE, AUDIO_SPEECH_CHUNK_SIZE, + BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME, BASE_MCP_ROUTE, DAILY_TAG_SPEND_BATCH_MULTIPLIER, DEFAULT_MAX_RECURSE_DEPTH, @@ -224,7 +234,7 @@ def generate_feedback_box(): import contextlib from collections import defaultdict from contextlib import asynccontextmanager -from functools import lru_cache +from functools import lru_cache, partial import litellm import litellm._redis @@ -402,6 +412,7 @@ from litellm.proxy.config_resolvers.alerting import ( ) from litellm.proxy.container_endpoints.endpoints import router as container_router from litellm.proxy.credential_endpoints.endpoints import router as credential_router +from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import ( SPEND_LOG_CLEANUP_BOUND_SETTINGS, SpendLogCleanup, @@ -3580,12 +3591,35 @@ async def _run_direct_health_check_with_instrumentation( raise AssertionError("perform_health_check rejected every optional argument") +async def _window_gated_health_check_db_save( + save: Callable[[], Awaitable[None]], + pod_lock_manager: PodLockManager | None, + lock_ttl: int | None, +) -> None: + """ + Persist at most once per window fleet-wide: the lock is the "this window's save is + done" marker, so it is deliberately never released and expires with the interval. + """ + if pod_lock_manager is not None and pod_lock_manager.redis_cache is not None: + acquired: Final = await pod_lock_manager.acquire_lock( + cronjob_id=BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME, + ttl=lock_ttl, + allow_reentrant=False, + ) + if not acquired: + verbose_proxy_logger.debug("background_health_check_db_save_skipped another pod persisted this window") + return + await save() + + def _schedule_background_health_check_db_save( prisma_client, shared_health_manager, model_list: list, healthy_endpoints: list, unhealthy_endpoints: list, + pod_lock_manager: PodLockManager | None = None, + lock_ttl: int | None = None, ): """Fire-and-forget: persist health check results to DB if prisma is available.""" if prisma_client is None: @@ -3598,16 +3632,16 @@ def _schedule_background_health_check_db_save( checked_by: Final = shared_health_manager.pod_id if shared_health_manager is not None else "background_health_check" start_time: Final = time_module.time() - asyncio.create_task( - _save_background_health_checks_to_db( - prisma_client, - model_list, - healthy_endpoints, - unhealthy_endpoints, - start_time, - checked_by=checked_by, - ) + save: Final = partial( + _save_background_health_checks_to_db, + prisma_client, + model_list, + healthy_endpoints, + unhealthy_endpoints, + start_time, + checked_by=checked_by, ) + asyncio.create_task(_window_gated_health_check_db_save(save, pod_lock_manager, lock_ttl)) def _get_endpoint_exception_status(endpoint: dict, exceptions: dict) -> int: @@ -3914,6 +3948,8 @@ async def _run_background_health_check(): _llm_model_list, healthy_endpoints, unhealthy_endpoints, + pod_lock_manager=proxy_logging_obj.db_spend_update_writer.pod_lock_manager, + lock_ttl=health_check_interval, ) # Write health state to router cache for health-check-driven routing diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index cab2bd6d9db..01530762800 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -121,6 +121,11 @@ from litellm.proxy.db.exception_handler import ( PrismaDBExceptionHandler, call_with_db_reconnect_retry, ) +from litellm.proxy.db.health_check_latest import ( + LatestHealthCheckRow, + fetch_latest_health_checks, + fetch_latest_health_checks_for_models, +) from litellm.proxy.db.log_db_metrics import log_db_metrics from litellm.proxy.db.prisma_client import ( PrismaWrapper, @@ -5962,48 +5967,13 @@ class PrismaClient: verbose_proxy_logger.error("Error getting health check history: %s", e) return [] - async def get_all_latest_health_checks(self) -> "Sequence[prisma_models.LiteLLM_HealthCheckTable]": - """ - Get the latest health check for each model. + async def get_all_latest_health_checks(self) -> tuple[LatestHealthCheckRow, ...]: + """Latest health check per (model_id, model_name), deduplicated in Postgres.""" + return await fetch_latest_health_checks(self) - Uses DB-level DISTINCT ON (model_id, model_name) with ORDER BY checked_at DESC - (via Prisma ``distinct`` + ``order``) so we never load the full history into memory. - """ - try: - return await HealthCheckRepository(self).table.find_many( - distinct=["model_id", "model_name"], - order=[ - {"model_id": "asc"}, - {"model_name": "asc"}, - {"checked_at": "desc"}, - ], - ) - except Exception as e: - verbose_proxy_logger.error("Error getting all latest health checks: %s", e) - return [] - - async def get_latest_health_checks_for_models( - self, model_names: "Sequence[str]" - ) -> "Sequence[prisma_models.LiteLLM_HealthCheckTable]": - """ - Get the latest health check for each of the named models. - - Same DISTINCT ON as ``get_all_latest_health_checks``, bounded to the models asked - about, so a paged caller reads health for its page instead of for the whole table. - """ - if not model_names: - return () - latest_first: Final = (("model_id", "asc"), ("model_name", "asc"), ("checked_at", "desc")) - order: Final = [{field: direction} for field, direction in latest_first] # mutable-ok: prisma order is a list - try: - return await HealthCheckRepository(self).table.find_many( - where={"model_name": {"in": list(model_names)}}, # mutable-ok: prisma filters are dicts and lists - distinct=["model_id", "model_name"], # mutable-ok: prisma distinct takes a list - order=order, - ) - except Exception as e: # noqa: BLE001 # health decorates a list; a driver error must not fail the page - verbose_proxy_logger.error("Error getting latest health checks for models: %s", e) - return () + async def get_latest_health_checks_for_models(self, model_names: Sequence[str]) -> tuple[LatestHealthCheckRow, ...]: + """Same as ``get_all_latest_health_checks``, bounded to the named models.""" + return await fetch_latest_health_checks_for_models(self, model_names) ### HELPER FUNCTIONS ### diff --git a/tests/test_litellm/proxy/db/test_health_check_latest.py b/tests/test_litellm/proxy/db/test_health_check_latest.py new file mode 100644 index 00000000000..529e4d8f2e9 --- /dev/null +++ b/tests/test_litellm/proxy/db/test_health_check_latest.py @@ -0,0 +1,111 @@ +from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.proxy.db.health_check_latest import ( + LATEST_HEALTH_CHECKS_FOR_MODELS_SQL, + LATEST_HEALTH_CHECKS_SQL, + fetch_latest_health_checks, + fetch_latest_health_checks_for_models, +) + + +def _prisma(rows): + prisma = MagicMock() + prisma.db.query_raw = AsyncMock(return_value=rows) + return prisma + + +def _raw_row(**overrides): + row = { + "health_check_id": "hc-1", + "model_name": "gpt-4", + "model_id": "deployment-abc", + "status": "healthy", + "healthy_count": 1, + "unhealthy_count": 0, + "error_message": None, + "response_time_ms": 12.5, + "details": None, + "checked_by": "pod-1", + "checked_at": "2026-08-25T00:00:00+00:00", + "created_at": "2026-08-25T00:00:00+00:00", + "updated_at": "2026-08-25T00:00:00+00:00", + } + return {**row, **overrides} + + +@pytest.mark.asyncio +async def test_fetch_all_runs_one_distinct_on_query_with_no_parameters(): + """The dedup must be in the SQL: prisma find_many(distinct=...) streams the whole history table.""" + prisma = _prisma([]) + assert await fetch_latest_health_checks(prisma) == () + assert prisma.db.query_raw.await_args.args == (LATEST_HEALTH_CHECKS_SQL,) + assert 'DISTINCT ON ("model_id", "model_name")' in LATEST_HEALTH_CHECKS_SQL + assert '"checked_at" DESC' in LATEST_HEALTH_CHECKS_SQL + + +@pytest.mark.asyncio +async def test_raw_datetimes_come_back_tz_aware_with_or_without_an_offset(): + """The save path subtracts checked_at from datetime.now(timezone.utc); a naive value would TypeError.""" + naive = _raw_row(health_check_id="hc-naive", model_id=None, checked_at="2026-08-25T00:00:00") + aware = _raw_row(health_check_id="hc-aware", checked_at="2026-08-25T01:00:00+02:00") + rows = await fetch_latest_health_checks(_prisma([naive, aware])) + assert {row.health_check_id: (row.model_id, row.checked_at) for row in rows} == { + "hc-naive": (None, datetime(2026, 8, 25, 0, 0, tzinfo=timezone.utc)), + "hc-aware": ("deployment-abc", datetime(2026, 8, 24, 23, 0, tzinfo=timezone.utc)), + } + + +@pytest.mark.asyncio +async def test_json_details_decode_from_text_and_pass_through_as_dict(): + rows = await fetch_latest_health_checks( + _prisma( + [ + _raw_row(health_check_id="text", details='{"region": "eu"}'), + _raw_row(health_check_id="dict", details={"region": "us"}), + _raw_row(health_check_id="none", details=None), + ] + ) + ) + assert {row.health_check_id: row.details for row in rows} == { + "text": {"region": "eu"}, + "dict": {"region": "us"}, + "none": None, + } + + +@pytest.mark.asyncio +async def test_fetch_all_degrades_to_no_rows_when_the_query_fails(): + prisma = _prisma([]) + prisma.db.query_raw.side_effect = RuntimeError("db down") + assert await fetch_latest_health_checks(prisma) == () + + +@pytest.mark.asyncio +async def test_fetch_all_degrades_to_no_rows_for_a_malformed_row(): + assert await fetch_latest_health_checks(_prisma([{"unexpected": "shape"}])) == () + + +@pytest.mark.asyncio +async def test_fetch_for_models_binds_the_page_as_the_only_parameter(): + prisma = _prisma([_raw_row()]) + rows = await fetch_latest_health_checks_for_models(prisma, ("gpt-4", "claude-opus")) + assert prisma.db.query_raw.await_args.args == (LATEST_HEALTH_CHECKS_FOR_MODELS_SQL, ["gpt-4", "claude-opus"]) + assert [row.model_name for row in rows] == ["gpt-4"] + assert 'WHERE "model_name" = ANY($1)' in LATEST_HEALTH_CHECKS_FOR_MODELS_SQL + + +@pytest.mark.asyncio +async def test_fetch_for_models_skips_the_database_for_an_empty_page(): + prisma = _prisma([]) + assert await fetch_latest_health_checks_for_models(prisma, ()) == () + prisma.db.query_raw.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_fetch_for_models_degrades_to_no_rows_when_the_query_fails(): + prisma = _prisma([]) + prisma.db.query_raw.side_effect = RuntimeError("db down") + assert await fetch_latest_health_checks_for_models(prisma, ("gpt-4",)) == () diff --git a/tests/test_litellm/proxy/proxy_server/test_background_health.py b/tests/test_litellm/proxy/proxy_server/test_background_health.py index 990844369f7..4874f75752c 100644 --- a/tests/test_litellm/proxy/proxy_server/test_background_health.py +++ b/tests/test_litellm/proxy/proxy_server/test_background_health.py @@ -20,6 +20,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest import litellm.proxy.proxy_server as proxy_server +from litellm.constants import BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME from litellm.proxy.proxy_server import ( _adaptive_router_flusher_loop, _get_endpoint_exception_status, @@ -245,6 +246,86 @@ async def test_schedule_background_health_check_db_save_invalid_no_event_loop_ra ) +def _lock_manager(redis_cache, acquired): + manager = MagicMock() + manager.redis_cache = redis_cache + manager.acquire_lock = AsyncMock(return_value=acquired) + manager.release_lock = AsyncMock() + return manager + + +def _capture_saves(monkeypatch): + saves = [] + + async def _fake_save(*_args, **kwargs): + saves.append(kwargs) + + import litellm.proxy.health_endpoints._health_endpoints as he + + monkeypatch.setattr(he, "_save_background_health_checks_to_db", _fake_save) + return saves + + +def _schedule_with(lock_manager): + _schedule_background_health_check_db_save( + prisma_client=MagicMock(), + shared_health_manager=None, + model_list=[], + healthy_endpoints=[], + unhealthy_endpoints=[], + pod_lock_manager=lock_manager, + lock_ttl=300, + ) + + +@pytest.mark.asyncio +async def test_schedule_background_health_check_db_save_skips_a_window_another_pod_persisted(monkeypatch): + saves = _capture_saves(monkeypatch) + lock_manager = _lock_manager(redis_cache=MagicMock(), acquired=False) + + _schedule_with(lock_manager) + await asyncio.sleep(0) + + assert saves == [] + + +@pytest.mark.asyncio +async def test_schedule_background_health_check_db_save_holds_the_window_lock_for_the_whole_interval(monkeypatch): + """The lock is the "saved this window" marker: never reentrant, TTL = interval, and never released.""" + saves = _capture_saves(monkeypatch) + lock_manager = _lock_manager(redis_cache=MagicMock(), acquired=True) + + _schedule_with(lock_manager) + await asyncio.sleep(0) + + assert normalize( + { + "saves": len(saves), + "lock_request": lock_manager.acquire_lock.await_args.kwargs, + "released": lock_manager.release_lock.await_count, + } + ) == { + "saves": 1, + "lock_request": { + "cronjob_id": BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME, + "ttl": 300, + "allow_reentrant": False, + }, + "released": 0, + } + + +@pytest.mark.asyncio +async def test_schedule_background_health_check_db_save_runs_ungated_without_redis(monkeypatch): + saves = _capture_saves(monkeypatch) + lock_manager = _lock_manager(redis_cache=None, acquired=True) + + _schedule_with(lock_manager) + await asyncio.sleep(0) + + assert (len(saves), lock_manager.acquire_lock.await_count) == (1, 0) + + # --------------------------------------------------------------------------- # _get_endpoint_exception_status # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/test_health_check_functions.py b/tests/test_litellm/proxy/test_health_check_functions.py index fdae11d517a..9adaddb032d 100644 --- a/tests/test_litellm/proxy/test_health_check_functions.py +++ b/tests/test_litellm/proxy/test_health_check_functions.py @@ -6,6 +6,8 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.db.health_check_latest import LatestHealthCheckRow from litellm.proxy.health_endpoints._health_endpoints import ( _aggregate_health_check_results, _build_model_param_to_info_mapping, @@ -13,6 +15,7 @@ from litellm.proxy.health_endpoints._health_endpoints import ( _save_background_health_checks_to_db, _save_health_check_results_if_changed, _save_health_check_to_db, + latest_health_checks_endpoint, ) from litellm.proxy.utils import PrismaClient @@ -451,96 +454,125 @@ async def test_save_background_health_checks_to_db_exception_handling(): # Function should complete without raising -@pytest.mark.asyncio -async def test_get_all_latest_health_checks_with_model_id(mock_prisma): - """Test get_all_latest_health_checks properly groups by model_id""" - mock_check2 = MagicMock() - mock_check2.model_id = "model-456" - mock_check2.model_name = "gpt-3.5-turbo" - mock_check2.checked_at = datetime.now(timezone.utc) - timedelta(minutes=5) - - mock_check3 = MagicMock() - mock_check3.model_id = "model-123" - mock_check3.model_name = "gpt-3.5-turbo" - mock_check3.checked_at = datetime.now(timezone.utc) - timedelta( - minutes=1 - ) # Latest for model-123 - - # Order by checked_at desc - mock_prisma.db.litellm_healthchecktable.find_many = AsyncMock( - return_value=[mock_check3, mock_check2] - ) - - result = await mock_prisma.get_all_latest_health_checks() - - # Should return 2 unique models (by model_id) - assert len(result) == 2 - - # Should have latest check for each model_id - model_ids = {check.model_id for check in result} - assert "model-123" in model_ids - assert "model-456" in model_ids - - # model-123 should have the latest check (1 minute ago) - model123_check = next(c for c in result if c.model_id == "model-123") - assert model123_check.checked_at == mock_check3.checked_at +def _raw_latest_row(model_name: str, model_id, checked_at: datetime) -> dict: + return { + "health_check_id": f"hc-{model_id or 'no-id'}-{model_name}", + "model_name": model_name, + "model_id": model_id, + "status": "healthy", + "healthy_count": 1, + "unhealthy_count": 0, + "error_message": None, + "response_time_ms": 10.0, + "details": None, + "checked_by": "pod-1", + "checked_at": checked_at.isoformat(), + "created_at": checked_at.isoformat(), + "updated_at": checked_at.isoformat(), + } @pytest.mark.asyncio -async def test_get_all_latest_health_checks_without_model_id(mock_prisma): - """Test get_all_latest_health_checks groups by model_name when model_id is None""" - mock_check2 = MagicMock() - mock_check2.model_id = None - mock_check2.model_name = "gpt-3.5-turbo" - mock_check2.checked_at = datetime.now(timezone.utc) - timedelta(minutes=1) # Latest - - mock_prisma.db.litellm_healthchecktable.find_many = AsyncMock( - return_value=[mock_check2] - ) - - result = await mock_prisma.get_all_latest_health_checks() - - # Should return 1 unique model (by model_name) - assert len(result) == 1 - assert result[0].model_name == "gpt-3.5-turbo" - assert result[0].checked_at == mock_check2.checked_at # Latest - - -@pytest.mark.asyncio -async def test_get_all_latest_health_checks_same_name_with_and_without_model_id( - mock_prisma, -): +async def test_get_all_latest_health_checks_keeps_every_distinct_group_with_its_own_checked_at(mock_prisma): """ - Same model_name can appear twice after DISTINCT ON: once keyed by (model_id, name) - and once by (NULL, name) — different Postgres groups than a single row with id. + Postgres owns the dedup. (id, name), (other id, name) and (NULL, name) are distinct groups and each row + must arrive typed, with its own checked_at, for the 1h re-save compare and the id-or-name lookup key. """ now = datetime.now(timezone.utc) - with_id = MagicMock() - with_id.model_id = "deployment-abc" - with_id.model_name = "gpt-4" - with_id.checked_at = now - timedelta(minutes=2) - - without_id = MagicMock() - without_id.model_id = None - without_id.model_name = "gpt-4" - without_id.checked_at = now - timedelta(minutes=1) - - mock_prisma.db.litellm_healthchecktable.find_many = AsyncMock( - return_value=[without_id, with_id] + mock_prisma.db.query_raw = AsyncMock( + return_value=[ + _raw_latest_row("gpt-3.5-turbo", "model-123", now - timedelta(minutes=1)), + _raw_latest_row("gpt-3.5-turbo", "model-456", now - timedelta(minutes=5)), + _raw_latest_row("gpt-4", "deployment-abc", now - timedelta(minutes=2)), + _raw_latest_row("gpt-4", None, now - timedelta(minutes=3)), + ] ) result = await mock_prisma.get_all_latest_health_checks() - assert len(result) == 2 - names = {r.model_name for r in result} - assert names == {"gpt-4"} - ids = {r.model_id for r in result} - assert "deployment-abc" in ids - assert None in ids + assert {(check.model_id, check.model_name): check.checked_at for check in result} == { + ("model-123", "gpt-3.5-turbo"): now - timedelta(minutes=1), + ("model-456", "gpt-3.5-turbo"): now - timedelta(minutes=5), + ("deployment-abc", "gpt-4"): now - timedelta(minutes=2), + (None, "gpt-4"): now - timedelta(minutes=3), + } - by_key = {(r.model_id, r.model_name): r for r in result} - assert by_key[("deployment-abc", "gpt-4")].checked_at == with_id.checked_at - assert by_key[(None, "gpt-4")].checked_at == without_id.checked_at + +@pytest.mark.asyncio +async def test_save_background_health_checks_compares_raw_checked_at_against_utc_now(mock_prisma): + """ + Raw rows carry ISO strings and the engine may omit the offset. A naive checked_at would TypeError + inside the 1h compare, be swallowed, and silently stop every save; a stale row must still re-save. + """ + stale = (datetime.now(timezone.utc) - timedelta(hours=2)).replace(tzinfo=None) + fresh = datetime.now(timezone.utc) - timedelta(minutes=5) + mock_prisma.db.query_raw = AsyncMock( + return_value=[ + _raw_latest_row("stale-model", "stale-id", stale), + _raw_latest_row("fresh-model", "fresh-id", fresh), + ] + ) + mock_prisma.save_health_check_result = AsyncMock() + model_list = [ + {"model_name": "stale-model", "model_info": {"id": "stale-id"}, "litellm_params": {"model": "openai/stale"}}, + {"model_name": "fresh-model", "model_info": {"id": "fresh-id"}, "litellm_params": {"model": "openai/fresh"}}, + ] + + await _save_background_health_checks_to_db( + mock_prisma, + model_list, + [{"model": "openai/stale"}, {"model": "openai/fresh"}], + [], + time.time(), + "pod-1", + ) + await asyncio.sleep(0) + + assert [call.kwargs["model_id"] for call in mock_prisma.save_health_check_result.await_args_list] == ["stale-id"] + + +@pytest.mark.asyncio +async def test_latest_health_checks_endpoint_serialises_raw_rows(monkeypatch): + row = LatestHealthCheckRow( + health_check_id="hc-1", + model_name="gpt-4", + model_id="deployment-abc", + status="healthy", + healthy_count=1, + unhealthy_count=0, + error_message=None, + response_time_ms=12.5, + details='{"region": "eu"}', + checked_by="pod-1", + checked_at=datetime(2026, 8, 25), + created_at=datetime(2026, 8, 25, tzinfo=timezone.utc), + updated_at=datetime(2026, 8, 25, tzinfo=timezone.utc), + ) + prisma = MagicMock() + prisma.get_all_latest_health_checks = AsyncMock(return_value=(row,)) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma) + + response = await latest_health_checks_endpoint(user_api_key_dict=UserAPIKeyAuth()) + + assert response == { + "latest_health_checks": { + "deployment-abc": { + "health_check_id": "hc-1", + "model_name": "gpt-4", + "model_id": "deployment-abc", + "status": "healthy", + "healthy_count": 1, + "unhealthy_count": 0, + "error_message": None, + "response_time_ms": 12.5, + "details": {"region": "eu"}, + "checked_by": "pod-1", + "checked_at": "2026-08-25T00:00:00+00:00", + "created_at": "2026-08-25T00:00:00+00:00", + } + }, + "total_models": 1, + } @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py index 9f48ba68b4f..fbe9934f06c 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py @@ -22,6 +22,10 @@ from unittest.mock import AsyncMock, MagicMock import pytest +from litellm.proxy.db.health_check_latest import ( + LATEST_HEALTH_CHECKS_FOR_MODELS_SQL, + LATEST_HEALTH_CHECKS_SQL, +) from litellm.proxy.utils import PrismaClient @@ -261,36 +265,49 @@ async def test_get_health_check_history_db_error_returns_empty_list( assert await prisma_client.get_health_check_history() == [] +def _raw_health_check_row(model_name: str = "gpt-4", model_id: str | None = "deployment-abc") -> dict[str, Any]: + return { + "health_check_id": f"hc-{model_name}", + "model_name": model_name, + "model_id": model_id, + "status": "healthy", + "healthy_count": 1, + "unhealthy_count": 0, + "error_message": None, + "response_time_ms": 12.5, + "details": None, + "checked_by": "pod-1", + "checked_at": "2026-08-25T00:00:00+00:00", + "created_at": "2026-08-25T00:00:00+00:00", + "updated_at": "2026-08-25T00:00:00+00:00", + } + + @pytest.mark.asyncio -async def test_get_all_latest_health_checks_uses_distinct( +async def test_get_all_latest_health_checks_dedups_in_postgres( prisma_client: PrismaClient, ) -> None: - rows = [MagicMock(name=f"row-{i}") for i in range(3)] - prisma_client.db.litellm_healthchecktable.find_many = AsyncMock(return_value=rows) + """A revert to prisma find_many(distinct=...) streams the whole history table; the SQL must own the DISTINCT.""" + prisma_client.db.query_raw = AsyncMock(return_value=[_raw_health_check_row()]) result = await prisma_client.get_all_latest_health_checks() - kwargs = prisma_client.db.litellm_healthchecktable.find_many.await_args.kwargs actual = { - "len": len(result), - "distinct": kwargs["distinct"], - "order_len": len(kwargs["order"]), - "first_order": kwargs["order"][0], + "query": prisma_client.db.query_raw.await_args.args, + "rows": [(row.model_id, row.model_name, row.status) for row in result], + "row_type": type(result[0]).__name__, } assert actual == { - "len": 3, - "distinct": ["model_id", "model_name"], - "order_len": 3, - "first_order": {"model_id": "asc"}, + "query": (LATEST_HEALTH_CHECKS_SQL,), + "rows": [("deployment-abc", "gpt-4", "healthy")], + "row_type": "LatestHealthCheckRow", } @pytest.mark.asyncio -async def test_get_all_latest_health_checks_db_error_returns_empty_list( +async def test_get_all_latest_health_checks_db_error_returns_no_rows( prisma_client: PrismaClient, ) -> None: - prisma_client.db.litellm_healthchecktable.find_many = AsyncMock( - side_effect=RuntimeError("oops") - ) - assert await prisma_client.get_all_latest_health_checks() == [] + prisma_client.db.query_raw = AsyncMock(side_effect=RuntimeError("oops")) + assert await prisma_client.get_all_latest_health_checks() == () @pytest.mark.asyncio @@ -298,18 +315,15 @@ async def test_get_latest_health_checks_for_models_bounds_the_query_to_those_mod prisma_client: PrismaClient, ) -> None: """A paged caller reads health for its page; an unbounded read is the bug this exists to avoid.""" - prisma_client.db.litellm_healthchecktable.find_many = AsyncMock(return_value=[]) - await prisma_client.get_latest_health_checks_for_models(["gpt-5", "claude-opus"]) - kwargs = prisma_client.db.litellm_healthchecktable.find_many.await_args.kwargs + prisma_client.db.query_raw = AsyncMock(return_value=[_raw_health_check_row(model_name="gpt-5")]) + result = await prisma_client.get_latest_health_checks_for_models(["gpt-5", "claude-opus"]) actual = { - "where": kwargs["where"], - "distinct": kwargs["distinct"], - "order": kwargs["order"], + "query": prisma_client.db.query_raw.await_args.args, + "rows": [row.model_name for row in result], } assert actual == { - "where": {"model_name": {"in": ["gpt-5", "claude-opus"]}}, - "distinct": ["model_id", "model_name"], - "order": [{"model_id": "asc"}, {"model_name": "asc"}, {"checked_at": "desc"}], + "query": (LATEST_HEALTH_CHECKS_FOR_MODELS_SQL, ["gpt-5", "claude-opus"]), + "rows": ["gpt-5"], } @@ -317,14 +331,14 @@ async def test_get_latest_health_checks_for_models_bounds_the_query_to_those_mod async def test_get_latest_health_checks_for_models_does_not_query_for_an_empty_page( prisma_client: PrismaClient, ) -> None: - prisma_client.db.litellm_healthchecktable.find_many = AsyncMock(return_value=[]) + prisma_client.db.query_raw = AsyncMock(return_value=[]) assert await prisma_client.get_latest_health_checks_for_models([]) == () - assert prisma_client.db.litellm_healthchecktable.find_many.await_count == 0 + assert prisma_client.db.query_raw.await_count == 0 @pytest.mark.asyncio -async def test_get_latest_health_checks_for_models_db_error_returns_empty_list( +async def test_get_latest_health_checks_for_models_db_error_returns_no_rows( prisma_client: PrismaClient, ) -> None: - prisma_client.db.litellm_healthchecktable.find_many = AsyncMock(side_effect=RuntimeError("oops")) + prisma_client.db.query_raw = AsyncMock(side_effect=RuntimeError("oops")) assert await prisma_client.get_latest_health_checks_for_models(["gpt-5"]) == () From f980b94923e25e5c01607fcdf2088bd0c232fc9e Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Thu, 3 Sep 2026 12:36:04 +0200 Subject: [PATCH 027/126] fix(proxy): release the health check save window lock on failure or cancel --- .../health_endpoints/_health_endpoints.py | 105 +++++++----- litellm/proxy/proxy_server.py | 45 +++-- .../proxy_server/test_background_health.py | 109 +++++++----- .../proxy/test_health_check_functions.py | 155 +++++++++++++----- 4 files changed, 278 insertions(+), 136 deletions(-) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 8b57bdca2fe..91825a5606c 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -7,7 +7,7 @@ import secrets import time import traceback from collections.abc import Iterable, Mapping -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Any, Final, Literal, TypedDict, cast import fastapi @@ -41,6 +41,7 @@ from litellm.proxy.auth.auth_utils import ( ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from litellm.proxy.db.health_check_latest import LatestHealthCheckRow from litellm.proxy.db.proxy_worker_heartbeat import count_live_proxy_workers from litellm.proxy.health_check import ( ADMIN_ONLY_HEALTH_DISPLAY_PARAMS, @@ -697,13 +698,42 @@ def _aggregate_health_check_results( return model_results +class _AggregatedHealthResult(TypedDict): + """One entry of ``_aggregate_health_check_results``: a model's counts for this cycle.""" + + model_name: ReadOnly[str] + model_id: ReadOnly[str | None] + healthy_count: ReadOnly[int] + unhealthy_count: ReadOnly[int] + error_message: ReadOnly[str | None] + + +def _new_health_status(result: _AggregatedHealthResult) -> str: + return "healthy" if result["healthy_count"] > 0 else "unhealthy" + + +def _should_persist_health_check_result( + result: _AggregatedHealthResult, latest_checks_map: Mapping[str, LatestHealthCheckRow] +) -> bool: + """ + True when this result has to be written: no previous row, the status changed, or the + previous row is older than one hour (periodic refresh while the status is stable). + """ + lookup_key: Final = result["model_id"] if result["model_id"] else result["model_name"] + last_check: Final = latest_checks_map.get(lookup_key) + if last_check is None or last_check.status != _new_health_status(result): + return True + time_since_last_check: Final = (datetime.now(timezone.utc) - last_check.checked_at).total_seconds() + return time_since_last_check >= 3600 # 1 hour threshold + + async def _save_health_check_results_if_changed( prisma_client, model_results: dict, latest_checks_map: dict, start_time: float, checked_by: str | None = None, -): +) -> bool: """ Save health check results to database, but only if status changed or >1 hour since last save. @@ -714,47 +744,39 @@ async def _save_health_check_results_if_changed( - Status changes: Immediate write (no delay) - Result: ~92% reduction in DB writes for stable systems, while maintaining real-time updates on changes + The writes are awaited rather than detached so the caller learns whether this cycle's + persistence completed. + Args: prisma_client: Database client model_results: Dictionary of aggregated health check results per model latest_checks_map: Dictionary mapping model_id/model_name to latest health check start_time: Start time of health check for calculating response time checked_by: Identifier for who/what performed the check + + Returns: + True when every row that needed writing was written (including when nothing needed + writing); False when any write failed. """ - for result in model_results.values(): - new_status = "healthy" if result["healthy_count"] > 0 else "unhealthy" - - # Check if we should save this result - should_save = True - lookup_key = result["model_id"] if result["model_id"] else result["model_name"] - if lookup_key in latest_checks_map: - last_check = latest_checks_map[lookup_key] - # Only save if status changed or if it's been a while since last check - if last_check.status == new_status: - # Check if last check was recent (within 1 hour) - if last_check.checked_at: - from datetime import datetime, timezone - - time_since_last_check = (datetime.now(timezone.utc) - last_check.checked_at).total_seconds() - # Only skip if status unchanged AND checked recently (within 1 hour) - # This ensures we still get periodic updates even if status is stable - if time_since_last_check < 3600: # 1 hour threshold - should_save = False - - if should_save: - asyncio.create_task( - prisma_client.save_health_check_result( - model_name=result["model_name"], - model_id=result["model_id"], - status=new_status, - healthy_count=result["healthy_count"], - unhealthy_count=result["unhealthy_count"], - error_message=result["error_message"], - response_time_ms=(time.time() - start_time) * 1000, - details=None, - checked_by=checked_by, - ) - ) + to_write: Final = tuple( + result for result in model_results.values() if _should_persist_health_check_result(result, latest_checks_map) + ) + writes: Final = tuple( + prisma_client.save_health_check_result( + model_name=result["model_name"], + model_id=result["model_id"], + status=_new_health_status(result), + healthy_count=result["healthy_count"], + unhealthy_count=result["unhealthy_count"], + error_message=result["error_message"], + response_time_ms=(time.time() - start_time) * 1000, + details=None, + checked_by=checked_by, + ) + for result in to_write + ) + rows: Final = await asyncio.gather(*writes) + return all(row is not None for row in rows) async def _save_background_health_checks_to_db( @@ -764,7 +786,7 @@ async def _save_background_health_checks_to_db( unhealthy_endpoints: list, start_time: float, checked_by: str | None = None, -): +) -> bool: """ Save background health check results to database for each model. @@ -773,9 +795,13 @@ async def _save_background_health_checks_to_db( OPTIMIZATION: Only saves to database if the status has changed from the last saved check. This dramatically reduces database writes when health status remains stable. + + Returns: + True when this cycle's persistence completed; False when it was skipped or any step + failed. Never raises: a database failure must not break the health check loop. """ if prisma_client is None: - return + return False try: # Step 1: Build mapping from model parameter to model info @@ -798,7 +824,7 @@ async def _save_background_health_checks_to_db( latest_checks_map[key] = check # Step 4: Save aggregated results, but only if status changed - await _save_health_check_results_if_changed( + return await _save_health_check_results_if_changed( prisma_client, model_results, latest_checks_map, @@ -808,6 +834,7 @@ async def _save_background_health_checks_to_db( except Exception as db_error: verbose_proxy_logger.warning("Failed to save background health checks to database: %s", db_error) # Continue execution - don't let database save failure break health checks + return False _PROXY_ADMIN_ROLES: Final = frozenset( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0e330272bdf..88ab36b3d1e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -152,6 +152,7 @@ if TYPE_CHECKING: from prisma import models as prisma_models from litellm.integrations.opentelemetry import OpenTelemetry + from litellm.proxy.health_check_utils.shared_health_check_manager import SharedHealthCheckManager Span = _Span | Any else: @@ -3592,35 +3593,49 @@ async def _run_direct_health_check_with_instrumentation( async def _window_gated_health_check_db_save( - save: Callable[[], Awaitable[None]], + save: Callable[[], Awaitable[bool]], pod_lock_manager: PodLockManager | None, lock_ttl: int | None, ) -> None: """ - Persist at most once per window fleet-wide: the lock is the "this window's save is - done" marker, so it is deliberately never released and expires with the interval. + Persist at most once per window fleet-wide. A completed save keeps the lock as the + "this window's save is done" marker, so it is deliberately never released and expires + with the interval. A save that reports failure or is cancelled releases the lock so + another pod's cycle in the same window can retry, instead of the fleet going a whole + window without a write. """ - if pod_lock_manager is not None and pod_lock_manager.redis_cache is not None: - acquired: Final = await pod_lock_manager.acquire_lock( - cronjob_id=BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME, - ttl=lock_ttl, - allow_reentrant=False, + if pod_lock_manager is None or pod_lock_manager.redis_cache is None: + await save() + return + acquired: Final = await pod_lock_manager.acquire_lock( + cronjob_id=BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME, + ttl=lock_ttl, + allow_reentrant=False, + ) + if not acquired: + verbose_proxy_logger.debug("background_health_check_db_save_skipped another pod persisted this window") + return + try: + persisted: Final = await save() + except BaseException: + await pod_lock_manager.release_lock(cronjob_id=BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME) + raise + if not persisted: + verbose_proxy_logger.warning( + "background_health_check_db_save_incomplete released the window lock so another pod can retry" ) - if not acquired: - verbose_proxy_logger.debug("background_health_check_db_save_skipped another pod persisted this window") - return - await save() + await pod_lock_manager.release_lock(cronjob_id=BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME) def _schedule_background_health_check_db_save( - prisma_client, - shared_health_manager, + prisma_client: PrismaClient | None, + shared_health_manager: "SharedHealthCheckManager | None", model_list: list, healthy_endpoints: list, unhealthy_endpoints: list, pod_lock_manager: PodLockManager | None = None, lock_ttl: int | None = None, -): +) -> None: """Fire-and-forget: persist health check results to DB if prisma is available.""" if prisma_client is None: return diff --git a/tests/test_litellm/proxy/proxy_server/test_background_health.py b/tests/test_litellm/proxy/proxy_server/test_background_health.py index 4874f75752c..d99aa637955 100644 --- a/tests/test_litellm/proxy/proxy_server/test_background_health.py +++ b/tests/test_litellm/proxy/proxy_server/test_background_health.py @@ -112,13 +112,11 @@ async def test_run_direct_health_check_with_instrumentation_returns_results( lambda _gs: {}, ) - healthy, unhealthy, exceptions = ( - await _run_direct_health_check_with_instrumentation( - model_list=[{"model_name": "gpt-4"}], - details=False, - max_concurrency=1, - instrumentation_context={"source": "test"}, - ) + healthy, unhealthy, exceptions = await _run_direct_health_check_with_instrumentation( + model_list=[{"model_name": "gpt-4"}], + details=False, + max_concurrency=1, + instrumentation_context={"source": "test"}, ) assert normalize( @@ -254,11 +252,12 @@ def _lock_manager(redis_cache, acquired): return manager -def _capture_saves(monkeypatch): +def _capture_saves(monkeypatch, persisted=True): saves = [] async def _fake_save(*_args, **kwargs): saves.append(kwargs) + return persisted import litellm.proxy.health_endpoints._health_endpoints as he @@ -266,6 +265,15 @@ def _capture_saves(monkeypatch): return saves +def _cancel_during_save(monkeypatch): + async def _fake_save(*_args, **_kwargs): + raise asyncio.CancelledError() + + import litellm.proxy.health_endpoints._health_endpoints as he + + monkeypatch.setattr(he, "_save_background_health_checks_to_db", _fake_save) + + def _schedule_with(lock_manager): _schedule_background_health_check_db_save( prisma_client=MagicMock(), @@ -315,15 +323,56 @@ async def test_schedule_background_health_check_db_save_holds_the_window_lock_fo } +@pytest.mark.asyncio +async def test_schedule_background_health_check_db_save_releases_the_window_lock_when_the_save_reports_failure( + monkeypatch, +): + """A failed save must not burn the window: release the lock so another pod's cycle can retry.""" + saves = _capture_saves(monkeypatch, persisted=False) + lock_manager = _lock_manager(redis_cache=MagicMock(), acquired=True) + + _schedule_with(lock_manager) + await asyncio.sleep(0) + + assert normalize( + { + "saves": len(saves), + "release_request": lock_manager.release_lock.await_args.kwargs, + "release_count": lock_manager.release_lock.await_count, + } + ) == { + "saves": 1, + "release_request": {"cronjob_id": BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME}, + "release_count": 1, + } + + +@pytest.mark.asyncio +async def test_schedule_background_health_check_db_save_releases_the_window_lock_when_the_save_is_cancelled( + monkeypatch, +): + """A pod shutting down mid-save releases the lock instead of holding it until the TTL.""" + _cancel_during_save(monkeypatch) + lock_manager = _lock_manager(redis_cache=MagicMock(), acquired=True) + + _schedule_with(lock_manager) + await asyncio.sleep(0) + + assert ( + lock_manager.release_lock.await_args.kwargs, + lock_manager.release_lock.await_count, + ) == ({"cronjob_id": BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME}, 1) + + @pytest.mark.asyncio async def test_schedule_background_health_check_db_save_runs_ungated_without_redis(monkeypatch): - saves = _capture_saves(monkeypatch) + saves = _capture_saves(monkeypatch, persisted=False) lock_manager = _lock_manager(redis_cache=None, acquired=True) _schedule_with(lock_manager) await asyncio.sleep(0) - assert (len(saves), lock_manager.acquire_lock.await_count) == (1, 0) + assert (len(saves), lock_manager.acquire_lock.await_count, lock_manager.release_lock.await_count) == (1, 0, 0) # --------------------------------------------------------------------------- @@ -400,13 +449,9 @@ def test_write_health_state_to_router_cache_sets_states(monkeypatch): _write_health_state_to_router_cache(healthy, unhealthy, exceptions) - fake_router.health_state_cache.set_deployment_health_states.assert_called_once_with( - fake_states - ) + fake_router.health_state_cache.set_deployment_health_states.assert_called_once_with(fake_states) - call_args = fake_router.health_state_cache.set_deployment_health_states.call_args[ - 0 - ][0] + call_args = fake_router.health_state_cache.set_deployment_health_states.call_args[0][0] assert normalize( { "states_keys": sorted(call_args.keys()), @@ -448,9 +493,7 @@ def test_write_health_state_to_router_cache_populates_for_listing_filter(monkeyp fake_router.cooldown_time = 30 monkeypatch.setattr(proxy_server, "llm_router", fake_router) - monkeypatch.setattr( - proxy_server, "general_settings", {"model_list_healthy_only": True} - ) + monkeypatch.setattr(proxy_server, "general_settings", {"model_list_healthy_only": True}) fake_states = {"m1": {"is_healthy": True}, "m2": {"is_healthy": False}} @@ -484,9 +527,7 @@ def test_write_health_state_to_router_cache_populates_for_listing_filter(monkeyp {"m2": SimpleNamespace(status_code=500)}, ) - fake_router.health_state_cache.set_deployment_health_states.assert_called_once_with( - fake_states - ) + fake_router.health_state_cache.set_deployment_health_states.assert_called_once_with(fake_states) assert cooldowns == [] assert failures == [] @@ -496,9 +537,7 @@ def test_write_health_state_to_router_cache_swallows_internal_failures(monkeypat fake_router = MagicMock() fake_router.enable_health_check_routing = True fake_router.health_check_ignore_transient_errors = False - fake_router.health_state_cache.set_deployment_health_states.side_effect = ( - RuntimeError("cache exploded") - ) + fake_router.health_state_cache.set_deployment_health_states.side_effect = RuntimeError("cache exploded") monkeypatch.setattr(proxy_server, "llm_router", fake_router) @@ -528,9 +567,7 @@ async def test_adaptive_router_flusher_loop_flushes_each_router(monkeypatch): from litellm.types.router import TaggedPreRoutingStrategy fake_router = MagicMock() - fake_router.adaptive_routers = { - "alpha": [TaggedPreRoutingStrategy(tags=(), strategy=fake_ar)] - } + fake_router.adaptive_routers = {"alpha": [TaggedPreRoutingStrategy(tags=(), strategy=fake_ar)]} monkeypatch.setattr(proxy_server, "llm_router", fake_router) monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) @@ -628,12 +665,8 @@ async def test_run_background_health_check_runs_one_cycle_then_cancels(monkeypat "_run_direct_health_check_with_instrumentation", _fake_direct, ) - monkeypatch.setattr( - proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None - ) - monkeypatch.setattr( - proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None - ) + monkeypatch.setattr(proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None) + monkeypatch.setattr(proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None) monkeypatch.setattr( proxy_server, "health_check_filter_kwargs_from_general_settings", @@ -711,12 +744,8 @@ async def test_run_background_health_check_probes_only_listed_model_groups(monke "_run_direct_health_check_with_instrumentation", _fake_direct, ) - monkeypatch.setattr( - proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None - ) - monkeypatch.setattr( - proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None - ) + monkeypatch.setattr(proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None) + monkeypatch.setattr(proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None) monkeypatch.setattr( proxy_server, "health_check_filter_kwargs_from_general_settings", diff --git a/tests/test_litellm/proxy/test_health_check_functions.py b/tests/test_litellm/proxy/test_health_check_functions.py index 9adaddb032d..5aa80213134 100644 --- a/tests/test_litellm/proxy/test_health_check_functions.py +++ b/tests/test_litellm/proxy/test_health_check_functions.py @@ -24,12 +24,8 @@ from litellm.proxy.utils import PrismaClient def mock_prisma(): """Simplified mock PrismaClient with bound methods""" client = MagicMock() - client.db.litellm_healthchecktable.create = AsyncMock( - return_value={"id": "test-id"} - ) - client.db.litellm_healthchecktable.find_many = AsyncMock( - return_value=[{"id": "1", "model_name": "test"}] - ) + client.db.litellm_healthchecktable.create = AsyncMock(return_value={"id": "test-id"}) + client.db.litellm_healthchecktable.find_many = AsyncMock(return_value=[{"id": "1", "model_name": "test"}]) # Bind actual methods import types @@ -55,14 +51,10 @@ def mock_prisma(): ("healthy", 1, 0, False), # Database error case ], ) -async def test_save_health_check_result( - mock_prisma, status, healthy, unhealthy, should_succeed -): +async def test_save_health_check_result(mock_prisma, status, healthy, unhealthy, should_succeed): """Test health check result saving with various scenarios""" if not should_succeed: - mock_prisma.db.litellm_healthchecktable.create.side_effect = Exception( - "DB Error" - ) + mock_prisma.db.litellm_healthchecktable.create.side_effect = Exception("DB Error") result = await mock_prisma.save_health_check_result( model_name="test-model", @@ -190,9 +182,7 @@ def test_aggregate_health_check_results(): {"model": "gpt-4", "error": "Rate limit exceeded"}, ] - result = _aggregate_health_check_results( - model_param_to_info, healthy_endpoints, unhealthy_endpoints - ) + result = _aggregate_health_check_results(model_param_to_info, healthy_endpoints, unhealthy_endpoints) # Check gpt-3.5-turbo is healthy gpt35_key = ("model-123", "gpt-3.5-turbo") @@ -223,9 +213,7 @@ def test_aggregate_health_check_results_multiple_endpoints(): ] unhealthy_endpoints = [] - result = _aggregate_health_check_results( - model_param_to_info, healthy_endpoints, unhealthy_endpoints - ) + result = _aggregate_health_check_results(model_param_to_info, healthy_endpoints, unhealthy_endpoints) key = ("model-123", "gpt-3.5-turbo") assert result[key]["healthy_count"] == 2 @@ -401,7 +389,7 @@ async def test_save_background_health_checks_to_db(): start_time = 1234567890.0 - await _save_background_health_checks_to_db( + persisted = await _save_background_health_checks_to_db( mock_prisma, model_list, healthy_endpoints, @@ -410,7 +398,8 @@ async def test_save_background_health_checks_to_db(): "background_health_check", ) - # Should call get_all_latest_health_checks and save_health_check_result + # Should call get_all_latest_health_checks and save_health_check_result, and report completion + assert persisted is True mock_prisma.get_all_latest_health_checks.assert_called_once() mock_prisma.save_health_check_result.assert_called_once() @@ -421,22 +410,112 @@ async def test_save_background_health_checks_to_db(): assert call_kwargs["checked_by"] == "background_health_check" +def _two_model_results(): + return { + ("model-1", "gpt-4"): { + "model_name": "gpt-4", + "model_id": "model-1", + "healthy_count": 1, + "unhealthy_count": 0, + "error_message": None, + }, + ("model-2", "gpt-4o"): { + "model_name": "gpt-4o", + "model_id": "model-2", + "healthy_count": 0, + "unhealthy_count": 1, + "error_message": "boom", + }, + } + + +@pytest.mark.asyncio +async def test_save_health_check_results_if_changed_awaits_every_write_and_reports_success(): + """Writes are awaited, not detached, so the caller can tell the cycle's persistence completed.""" + mock_prisma = MagicMock() + mock_prisma.save_health_check_result = AsyncMock(return_value={"id": "row"}) + + persisted = await _save_health_check_results_if_changed( + mock_prisma, _two_model_results(), {}, 1234567890.0, "background_health_check" + ) + + assert (persisted, mock_prisma.save_health_check_result.await_count) == (True, 2) + + +@pytest.mark.asyncio +async def test_save_health_check_results_if_changed_reports_failure_when_a_write_returns_none(): + """save_health_check_result swallows DB errors and returns None; that must surface as False.""" + mock_prisma = MagicMock() + mock_prisma.save_health_check_result = AsyncMock(side_effect=[{"id": "row"}, None]) + + persisted = await _save_health_check_results_if_changed( + mock_prisma, _two_model_results(), {}, 1234567890.0, "background_health_check" + ) + + assert (persisted, mock_prisma.save_health_check_result.await_count) == (False, 2) + + +@pytest.mark.asyncio +async def test_save_health_check_results_if_changed_reports_success_when_nothing_needed_writing(): + mock_prisma = MagicMock() + mock_prisma.save_health_check_result = AsyncMock() + model_results = { + ("model-1", "gpt-4"): { + "model_name": "gpt-4", + "model_id": "model-1", + "healthy_count": 1, + "unhealthy_count": 0, + "error_message": None, + }, + } + latest_checks_map = { + "model-1": MagicMock(status="healthy", checked_at=datetime.now(timezone.utc) - timedelta(minutes=5)), + } + + persisted = await _save_health_check_results_if_changed( + mock_prisma, model_results, latest_checks_map, 1234567890.0, "background_health_check" + ) + + assert (persisted, mock_prisma.save_health_check_result.await_count) == (True, 0) + + +def _one_model_setup(): + model_list = [ + { + "model_name": "gpt-3.5-turbo", + "model_info": {"id": "model-123"}, + "litellm_params": {"model": "gpt-3.5-turbo"}, + }, + ] + return model_list, [{"model": "gpt-3.5-turbo"}], [] + + +@pytest.mark.asyncio +async def test_save_background_health_checks_to_db_returns_false_when_a_write_fails(): + mock_prisma = MagicMock() + mock_prisma.get_all_latest_health_checks = AsyncMock(return_value=[]) + mock_prisma.save_health_check_result = AsyncMock(return_value=None) + model_list, healthy_endpoints, unhealthy_endpoints = _one_model_setup() + + persisted = await _save_background_health_checks_to_db( + mock_prisma, model_list, healthy_endpoints, unhealthy_endpoints, 1234567890.0, "background_health_check" + ) + + assert (persisted, mock_prisma.save_health_check_result.await_count) == (False, 1) + + @pytest.mark.asyncio async def test_save_background_health_checks_to_db_no_prisma(): """Test graceful handling when no prisma client""" - result = await _save_background_health_checks_to_db( - None, [], [], [], 0.0, "background_health_check" - ) - assert result is None + result = await _save_background_health_checks_to_db(None, [], [], [], 0.0, "background_health_check") + assert result is False @pytest.mark.asyncio async def test_save_background_health_checks_to_db_exception_handling(): """Test exception handling in background health check save""" mock_prisma = MagicMock() - mock_prisma.get_all_latest_health_checks = AsyncMock( - side_effect=Exception("DB Error") - ) + mock_prisma.get_all_latest_health_checks = AsyncMock(side_effect=Exception("DB Error")) model_list = [ { @@ -446,12 +525,13 @@ async def test_save_background_health_checks_to_db_exception_handling(): }, ] - # Should not raise exception, should handle gracefully - await _save_background_health_checks_to_db( + # Must not raise (the health check loop has to survive a DB outage) but must report + # the failure, so the window lock can be released for another pod to retry + persisted = await _save_background_health_checks_to_db( mock_prisma, model_list, [], [], 0.0, "background_health_check" ) - # Function should complete without raising + assert persisted is False def _raw_latest_row(model_name: str, model_id, checked_at: datetime) -> dict: @@ -660,12 +740,7 @@ def test_parse_background_health_check_model_groups_unset_returns_none(): assert parse_background_health_check_model_groups(None) is None assert parse_background_health_check_model_groups({}) is None - assert ( - parse_background_health_check_model_groups( - {"background_health_check_model_groups": None} - ) - is None - ) + assert parse_background_health_check_model_groups({"background_health_check_model_groups": None}) is None def test_parse_background_health_check_model_groups_list_returns_frozenset(): @@ -682,9 +757,7 @@ def test_parse_background_health_check_model_groups_malformed_raises(bad_value): from litellm.proxy.health_check import parse_background_health_check_model_groups with pytest.raises(ValueError, match="must be a list of model group names"): - parse_background_health_check_model_groups( - {"background_health_check_model_groups": bad_value} - ) + parse_background_health_check_model_groups({"background_health_check_model_groups": bad_value}) def test_filter_deployments_to_model_groups(): @@ -697,9 +770,7 @@ def test_filter_deployments_to_model_groups(): ] assert filter_deployments_to_model_groups(model_list, None) == tuple(model_list) - assert filter_deployments_to_model_groups( - model_list, frozenset({"prod-openai"}) - ) == (model_list[0], model_list[2]) + assert filter_deployments_to_model_groups(model_list, frozenset({"prod-openai"})) == (model_list[0], model_list[2]) assert filter_deployments_to_model_groups(model_list, frozenset()) == () From 7c1d060b7af3150320a2c88428871703d71f7a38 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:04:36 -0700 Subject: [PATCH 028/126] fix(mcp): enforce OAuth identity binding across credential lifetime --- litellm/proxy/_experimental/mcp_server/db.py | 34 +++- .../mcp_server/discoverable_endpoints.py | 94 +++++++--- .../mcp_server/oauth_identity_binding.py | 170 ++++++++++++++---- .../authz_code_refresher.py | 37 +++- .../per_user_oauth_store.py | 13 ++ .../types/mcp_server/mcp_server_manager.py | 13 +- .../test_authz_code_refresher.py | 34 ++++ .../test_per_user_oauth_store.py | 25 +++ .../mcp_server/test_db_credentials.py | 24 ++- .../mcp_server/test_discoverable_endpoints.py | 60 ++++++- .../mcp_server/test_oauth_identity_binding.py | 111 +++++++++--- .../test_mcp_management_endpoints.py | 3 +- 12 files changed, 526 insertions(+), 92 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 7379126983a..08f83aeb756 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -6,10 +6,17 @@ from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, cast +from typing_extensions import ReadOnly + from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( + RefreshTokenPresented, + credential_binding_matches, + enforce_oauth_identity_binding, +) from litellm.proxy._experimental.mcp_server.oauth_utils import build_upstream_oauth2_token_request from litellm.proxy._types import ( LiteLLM_MCPServerTable, @@ -117,6 +124,7 @@ class _OAuthCredentialAccessToken(TypedDict): class OAuthCredentialPayload(_OAuthCredentialAccessToken, total=False): + identity_binding_proof: ReadOnly[str] type: str refresh_token: str expires_at: str @@ -1393,6 +1401,7 @@ async def store_user_oauth_credential( expires_in: int | None = None, scopes: list[str] | None = None, skip_byok_guard: bool = False, + identity_binding_proof: str | None = None, ) -> None: """Persist an OAuth2 access token for a user+server pair. @@ -1409,6 +1418,7 @@ async def store_user_oauth_credential( "type": "oauth2", "access_token": access_token, "connected_at": datetime.now(timezone.utc).isoformat(), + **({"identity_binding_proof": identity_binding_proof} if identity_binding_proof else {}), } if refresh_token: payload["refresh_token"] = refresh_token @@ -1628,6 +1638,11 @@ async def refresh_user_oauth_token( warning and returns ``None`` — the caller is responsible for clearing the stale credential and triggering re-authentication. """ + binding: Final = server.oauth_identity_binding + if binding is not None and binding.mode == "enforce": + if not await credential_binding_matches(binding, user_id, server.server_id, cred): + return None + refresh_token: Final[str | None] = cred.get("refresh_token") token_url: Final[str | None] = getattr(server, "effective_token_url", None) or getattr(server, "token_url", None) server_id: Final[str] = getattr(server, "server_id", "") @@ -1677,6 +1692,14 @@ async def refresh_user_oauth_token( ) return None + binding_proof: Final = await enforce_oauth_identity_binding( + server=server, + token_response=body, + litellm_user_id=user_id, + grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented(refresh_token), + ) + access_token: Final[str | None] = body.get("access_token") if not access_token: verbose_proxy_logger.warning( @@ -1709,6 +1732,7 @@ async def refresh_user_oauth_token( refresh_token=new_refresh_token, expires_in=expires_in, scopes=scopes, + identity_binding_proof=binding_proof, skip_byok_guard=True, # Row is already OAuth2; skip the extra find_unique check ) @@ -1742,6 +1766,10 @@ async def resolve_valid_user_oauth_token( grant: Final = oauth_grant_state(cred) if cred is None or grant == "absent": return None + binding: Final = server.oauth_identity_binding + if binding is not None and binding.mode == "enforce": + if not await credential_binding_matches(binding, user_id, server.server_id, cred): + return None if grant == "valid": return cred if prisma_client is None: @@ -1782,7 +1810,11 @@ async def resolve_user_oauth_access_token( mcp_per_user_token_cache, ) - if prefetched_creds is None: + binding: Final = server.oauth_identity_binding + enforce_binding: Final = binding is not None and binding.mode == "enforce" + if enforce_binding: + await mcp_per_user_token_cache.delete(user_id, server_id) + if prefetched_creds is None and not enforce_binding: cached_token: Final = await mcp_per_user_token_cache.get(user_id, server_id) if cached_token is not None: return cached_token diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 9f6bc1860e1..c8e02be308b 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -144,6 +144,7 @@ def encode_state_with_base_url( dcr_client_id: str | None = None, dcr_client_secret: str | None = None, dcr_token_endpoint_auth_method: MCPTokenEndpointAuthMethod | None = None, + oauth_nonce: str | None = None, ) -> str: """ Encode the base_url, original state, and PKCE parameters using encryption. @@ -154,9 +155,8 @@ def encode_state_with_base_url( code_challenge: PKCE code challenge from client code_challenge_method: PKCE code challenge method from client client_redirect_uri: Original redirect_uri from client - litellm_user_id: The SSO-authenticated litellm user captured at the bridge authorize - (interactive dcr_bridge oauth_delegate only); the callback seals it into the gateway - authorization code so the token mint can bind the envelope to this user + litellm_user_id: The authenticated user captured for bridge or identity-bound per-user OAuth; + the callback seals this credential owner into the authorization code mcp_server_id: The server the flow targets, sealed alongside litellm_user_id (bridge) or dcr_client_id (ephemeral mint) so the gateway code cannot be replayed against another server @@ -174,6 +174,7 @@ def encode_state_with_base_url( An encrypted string that encodes all values """ state_data: Final = { + "oauth_nonce": oauth_nonce, "base_url": base_url, "original_state": original_state, "code_challenge": code_challenge, @@ -215,10 +216,10 @@ _BRIDGE_AUTH_CODE_PREFIX: Final = "llm_bcode_" class _BridgeAuthorizationCode(BaseModel): - """The identity and upstream code the gateway seals into the authorization code it hands a DCR - client for an interactive dcr_bridge oauth_delegate sign-in, recovered at the token endpoint.""" + """Authenticated caller and upstream code sealed for bridge or identity-bound per-user OAuth.""" model_config = ConfigDict(frozen=True) + oauth_nonce: str | None = None upstream_code: str = Field(min_length=1) litellm_user_id: str = Field(min_length=1) mcp_server_id: str = Field(min_length=1) @@ -230,7 +231,12 @@ def is_bridge_authorization_code(code: str) -> bool: return code.startswith(_BRIDGE_AUTH_CODE_PREFIX) -def seal_bridge_authorization_code(upstream_code: str, litellm_user_id: str, mcp_server_id: str) -> str: +def seal_bridge_authorization_code( + upstream_code: str, + litellm_user_id: str, + mcp_server_id: str, + oauth_nonce: str | None = None, +) -> str: """Seal the upstream authorization code and the SSO-captured litellm user into a gateway authorization code. The DCR client only echoes this opaque value back at the token endpoint; the gateway decrypts it there to recover the user (to bind the envelope) and the upstream code (to @@ -239,7 +245,12 @@ def seal_bridge_authorization_code(upstream_code: str, litellm_user_id: str, mcp authenticated symmetric helper (the same family the OAuth state uses), so the client can neither read nor forge it.""" payload: Final = json.dumps( - {"upstream_code": upstream_code, "litellm_user_id": litellm_user_id, "mcp_server_id": mcp_server_id}, + { + "upstream_code": upstream_code, + "litellm_user_id": litellm_user_id, + "mcp_server_id": mcp_server_id, + "oauth_nonce": oauth_nonce, + }, sort_keys=True, ) return _BRIDGE_AUTH_CODE_PREFIX + encrypt_value_helper(payload) @@ -552,6 +563,7 @@ async def _store_per_user_token_server_side( server: MCPServer, user_id: str, token_response: dict[str, Any], + identity_binding_proof: str | None = None, ) -> None: """Persist the OAuth token server-side and warm the Redis cache. @@ -593,6 +605,7 @@ async def _store_per_user_token_server_side( refresh_token=refresh_token, expires_in=expires_in, scopes=scopes, + identity_binding_proof=identity_binding_proof, ) verbose_logger.info( "_store_per_user_token_server_side: stored token for user=%s server=%s", @@ -859,6 +872,11 @@ async def authorize_with_server( ), ) + binding: Final = resolved_server.oauth_identity_binding + enforce_binding: Final = binding is not None and binding.mode == "enforce" + if enforce_binding: + _require_s256_pkce(code_challenge, code_challenge_method) + if resolved_server.is_dcr_bridge: # Enforce S256 PKCE on both bridge arms. The relay arm forwards the validated, # now-non-optional pair to the upstream authorize; the short-circuit arm keeps @@ -889,19 +907,16 @@ async def authorize_with_server( base_url: Final = urlunparse(parsed._replace(query="")) request_base_url: Final = get_request_base_url(request) - # Interactive dcr_bridge oauth_delegate sign-in: this arm runs the gateway /callback and /token in - # the loop, so the gateway can capture the litellm user here (from the browser's UI session) and - # carry it to the back-channel token mint. Seal the SSO user and the target server into the state; - # the callback reads them back to mint the gateway authorization code. A DCR client cannot present a - # litellm key, so the browser session is the only identity source; without one there is nothing to - # bind, so send the user through login first. Every other oauth2 server keeps the identity-less state. + # Seal the authenticated caller into state so the token exchange cannot select another credential owner. litellm_user_id: str | None = None - if resolved_server.is_dcr_bridge and resolved_server.is_oauth_delegate: + if enforce_binding or (resolved_server.is_dcr_bridge and resolved_server.is_oauth_delegate): from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( # noqa: PLC0415 # inline import avoids a module-load circular import _user_id_from_session_cookie, ) - litellm_user_id = _user_id_from_session_cookie(request) + litellm_user_id = ( + await _extract_user_id_from_request(request) if enforce_binding else None + ) or _user_id_from_session_cookie(request) if litellm_user_id is None: return _redirect_to_litellm_login(request) denial: Final = await _bridge_authorize_access_denial( @@ -913,9 +928,11 @@ async def authorize_with_server( if denial is not None: return denial + oauth_nonce: Final = secrets.token_urlsafe(32) if enforce_binding else None encoded_state: Final = encode_state_with_base_url( base_url=base_url, original_state=state, + oauth_nonce=oauth_nonce, code_challenge=code_challenge, code_challenge_method=code_challenge_method, client_redirect_uri=redirect_uri, @@ -935,11 +952,16 @@ async def authorize_with_server( "state": relay_state, "response_type": response_type or "code", } + if oauth_nonce: + params["nonce"] = oauth_nonce if scope: params["scope"] = scope elif resolved_server.scopes: params["scope"] = " ".join(resolved_server.scopes) + if enforce_binding and "openid" not in params.get("scope", "").split(): + params["scope"] = f"openid {params.get('scope', '')}".strip() + if code_challenge: params["code_challenge"] = code_challenge if code_challenge_method: @@ -1020,6 +1042,12 @@ async def exchange_token_with_server( except TokenEndpointAuthConfigError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc + request_user_id: Final = ( + await _extract_user_id_from_request(request) + if resolved_server.needs_user_oauth_token or resolved_server.oauth_identity_binding is not None + else None + ) + bridge_identity: _BridgeAuthorizationCode | None = None bridge_mint_ready: _BridgeMintReady | None = None bridge_upstream_refresh: SecretStr | None = None @@ -1081,6 +1109,14 @@ async def exchange_token_with_server( detail="Authorization code was issued for a different MCP server", ) code = bridge_identity.upstream_code + binding: Final = resolved_server.oauth_identity_binding + if binding is not None and binding.mode == "enforce": + if bridge_identity is None or not bridge_identity.oauth_nonce: + raise HTTPException(status_code=403, detail={"error": "oauth_identity_binding_failed"}) + if request_user_id is not None and request_user_id != bridge_identity.litellm_user_id: + raise HTTPException(status_code=403, detail={"error": "oauth_principal_mismatch"}) + if not code_verifier: + raise HTTPException(status_code=403, detail={"error": "oauth_identity_binding_failed"}) bridge_token_relay: Final = _dcr_bridge_relays_client_registration(resolved_server) if bridge_token_relay and not redirect_uri: raise HTTPException( @@ -1108,6 +1144,16 @@ async def exchange_token_with_server( return _bridge_mint_error_response(prepared) bridge_mint_ready = prepared + refresh_binding: Final = resolved_server.oauth_identity_binding + if grant_type == "refresh_token" and refresh_binding is not None and refresh_binding.mode == "enforce": + await enforce_oauth_identity_binding( + server=resolved_server, + token_response={}, + litellm_user_id=request_user_id, + grant_type=grant_type, + refresh_ownership=refresh_ownership, + ) + async_client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) try: response: Final = await async_client.post( @@ -1150,19 +1196,19 @@ async def exchange_token_with_server( # Bind the exchanged token to the LiteLLM caller BEFORE it is returned, stored, or cached, so a # token minted for a different upstream principal never becomes usable under the caller's user_id. - resolved_user_id: Final = ( - await _extract_user_id_from_request(request) - if resolved_server.needs_user_oauth_token or resolved_server.oauth_identity_binding is not None - else None - ) - if isinstance(token_response, dict): + resolved_user_id: Final = bridge_identity.litellm_user_id if bridge_identity else request_user_id + binding_proof: Final = ( await enforce_oauth_identity_binding( server=resolved_server, token_response=token_response, litellm_user_id=resolved_user_id, grant_type=grant_type, refresh_ownership=refresh_ownership, + expected_nonce=bridge_identity.oauth_nonce if bridge_identity else None, ) + if isinstance(token_response, dict) + else None + ) # Store server-side when the server is configured for per-user OAuth and # the calling client has provided a valid LiteLLM identity. @@ -1175,6 +1221,7 @@ async def exchange_token_with_server( server=resolved_server, user_id=user_id, token_response=token_response, + identity_binding_proof=binding_proof, ) except Exception as exc: verbose_logger.warning( @@ -2161,7 +2208,10 @@ async def callback( forwarded_code = code if isinstance(litellm_user_id, str) and litellm_user_id and isinstance(mcp_server_id, str) and mcp_server_id: forwarded_code = seal_bridge_authorization_code( - upstream_code=code, litellm_user_id=litellm_user_id, mcp_server_id=mcp_server_id + upstream_code=code, + litellm_user_id=litellm_user_id, + mcp_server_id=mcp_server_id, + oauth_nonce=state_data.get("oauth_nonce"), ) elif isinstance(dcr_client_id, str) and dcr_client_id and isinstance(mcp_server_id, str) and mcp_server_id: forwarded_code = seal_passthrough_authorization_code( diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py index 58ec3cfdb4f..7ca9e6a23df 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -8,9 +8,12 @@ claim is compared to the caller's trusted LiteLLM identity. Mismatches fail clos and are logged in audit mode. """ +import hashlib +import hmac +import json from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass -from typing import Final, Literal, TypeAlias +from typing import Final, Literal, Protocol, TypeAlias import jwt from fastapi import HTTPException @@ -45,9 +48,17 @@ CallerPrincipalLoader: TypeAlias = Callable[ [str, MCPOAuthIdentityBinding], # mutable-ok: Callable parameter syntax requires a list Awaitable[str | None], ] + + +@dataclass(frozen=True, slots=True) +class VerifiedRefreshToken: + refresh_token: str + binding_proof: str + + StoredRefreshTokenLoader: TypeAlias = Callable[ - [str, str], # mutable-ok: Callable parameter syntax requires a list - Awaitable[str | None], + [str, str, MCPOAuthIdentityBinding], # mutable-ok: Callable parameter syntax requires a list + Awaitable[VerifiedRefreshToken | None], ] _RejectionCode: TypeAlias = Literal["oauth_principal_mismatch", "oauth_identity_binding_failed"] @@ -72,6 +83,18 @@ class RefreshTokenPresented: RefreshOwnership: TypeAlias = RefreshOwnershipProven | RefreshTokenPresented | None +class BindingValidator(Protocol): + async def __call__( + self, + *, + server: MCPServer, + token_response: Mapping[str, object], + litellm_user_id: str | None, + grant_type: str, + refresh_ownership: RefreshOwnership, + ) -> str | None: ... + + async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> Sequence[Mapping[str, object]]: jwks_url: Final[str] = binding.jwks_url or await _discover_jwks_url(binding.issuer) cached: Final = await _jwks_cache.async_get_cache(jwks_url) @@ -118,7 +141,7 @@ def _decode_id_token( signing_key: "jwt.PyJWK", ) -> "Mapping[str, object] | _BindingRejection": try: - decode_options: Final[Options] = {"require": ("iss", "exp")} + decode_options: Final[Options] = {"require": ("iss", "exp", "aud", "sub", "iat")} return jwt.decode( id_token, signing_key.key, @@ -169,7 +192,9 @@ async def _load_caller_principal(litellm_user_id: str, binding: MCPOAuthIdentity return loaded.user_email -async def _load_stored_refresh_token(litellm_user_id: str, server_id: str) -> str | None: +async def _load_stored_refresh_token( + litellm_user_id: str, server_id: str, binding: MCPOAuthIdentityBinding +) -> VerifiedRefreshToken | None: try: from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 # keep database imports lazy get_user_oauth_credential, @@ -184,17 +209,89 @@ async def _load_stored_refresh_token(litellm_user_id: str, server_id: str) -> st user_id=litellm_user_id, server_id=server_id, ) - return cred.get("refresh_token") if cred else None + if not cred or not await credential_binding_matches(binding, litellm_user_id, server_id, cred): + return None + refresh_token: Final = cred.get("refresh_token") + proof: Final = cred.get("identity_binding_proof") + return VerifiedRefreshToken(refresh_token, proof) if refresh_token and proof else None except Exception: # noqa: BLE001 # a credential lookup failure must fail closed return None +async def current_binding_proof( + binding: MCPOAuthIdentityBinding, + user_id: str, + server_id: str, + caller_principal_loader: CallerPrincipalLoader = _load_caller_principal, +) -> str | None: + principal: Final = await caller_principal_loader(user_id, binding) + if not principal: + return None + return _binding_proof(binding, user_id, server_id, principal) + + +def _binding_proof(binding: MCPOAuthIdentityBinding, user_id: str, server_id: str, principal: str) -> str: + payload: Final = json.dumps( + ("oidc-nonce-v1", server_id, user_id, principal, binding.model_dump(mode="json")), + sort_keys=True, + ) + return hashlib.sha256(payload.encode()).hexdigest() + + +async def credential_binding_matches( + binding: MCPOAuthIdentityBinding, + user_id: str, + server_id: str, + credential: Mapping[str, object], + caller_principal_loader: CallerPrincipalLoader = _load_caller_principal, +) -> bool: + stored: Final = credential.get("identity_binding_proof") + if not isinstance(stored, str) or not stored: + return False + expected: Final = await current_binding_proof(binding, user_id, server_id, caller_principal_loader) + return expected is not None and hmac.compare_digest(stored, expected) + + def _principals_match(upstream: str, caller: str, binding: MCPOAuthIdentityBinding) -> bool: if binding.principal_claim == "email" or binding.caller_field == "user_email": return upstream.strip().casefold() == caller.strip().casefold() return upstream == caller +async def _evaluate_refresh_ownership( + binding: MCPOAuthIdentityBinding, + litellm_user_id: str | None, + server_id: str, + refresh_ownership: RefreshOwnership, + stored_refresh_token_loader: StoredRefreshTokenLoader, +) -> _BindingRejection | str: + match refresh_ownership: + case RefreshOwnershipProven(): + return _BindingRejection( + code="oauth_identity_binding_failed", + description="an identity envelope alone does not prove upstream principal binding", + ) + case None: + return _BindingRejection( + code="oauth_identity_binding_failed", + description="refresh_token grant without an id_token carries no refresh token to prove ownership of", + ) + case RefreshTokenPresented(refresh_token): + if not litellm_user_id: + return _BindingRejection( + code="oauth_identity_binding_failed", + description="the request carries no resolvable LiteLLM user identity to bind the credential to", + ) + stored: Final = await stored_refresh_token_loader(litellm_user_id, server_id, binding) + if stored is None or not hmac.compare_digest(stored.refresh_token, refresh_token): + return _BindingRejection( + code="oauth_identity_binding_failed", + description="the presented refresh_token is not the caller's stored credential for this server", + ) + return stored.binding_proof + assert_never(refresh_ownership) # pragma: no cover + + async def _evaluate_binding( binding: MCPOAuthIdentityBinding, token_response: Mapping[str, object], @@ -205,7 +302,8 @@ async def _evaluate_binding( jwks_fetcher: JwksFetcher, caller_principal_loader: CallerPrincipalLoader, stored_refresh_token_loader: StoredRefreshTokenLoader, -) -> _BindingRejection | None: + expected_nonce: str | None, +) -> _BindingRejection | str: id_token: Final = token_response.get("id_token") if not isinstance(id_token, str) or not id_token: if grant_type != "refresh_token": @@ -213,28 +311,13 @@ async def _evaluate_binding( code="oauth_identity_binding_failed", description="the upstream token response carries no id_token to bind the credential to a principal", ) - match refresh_ownership: - case RefreshOwnershipProven(): - return None - case None: - return _BindingRejection( - code="oauth_identity_binding_failed", - description="refresh_token grant without an id_token carries no refresh token to prove ownership of", - ) - case RefreshTokenPresented(refresh_token): - if not litellm_user_id: - return _BindingRejection( - code="oauth_identity_binding_failed", - description="the request carries no resolvable LiteLLM user identity to bind the credential to", - ) - stored: Final = await stored_refresh_token_loader(litellm_user_id, server_id) - if stored is None or stored != refresh_token: - return _BindingRejection( - code="oauth_identity_binding_failed", - description="the presented refresh_token is not the caller's stored credential for this server", - ) - return None - assert_never(refresh_ownership) # pragma: no cover + return await _evaluate_refresh_ownership( + binding, + litellm_user_id, + server_id, + refresh_ownership, + stored_refresh_token_loader, + ) if not litellm_user_id: return _BindingRejection( code="oauth_identity_binding_failed", @@ -247,12 +330,25 @@ async def _evaluate_binding( code="oauth_identity_binding_failed", description=f"could not fetch the issuer's JWKS: {exc}", ) - signing_key: Final = _select_signing_key(id_token, keys) + try: + signing_key: Final = _select_signing_key(id_token, keys) + except (jwt.PyJWTError, ValueError, TypeError): + return _BindingRejection( + code="oauth_identity_binding_failed", + description="invalid id_token header or issuer signing key", + ) if isinstance(signing_key, _BindingRejection): return signing_key claims: Final = _decode_id_token(id_token, binding, signing_key) if isinstance(claims, _BindingRejection): return claims + if grant_type == "authorization_code": + nonce: Final = claims.get("nonce") + if not expected_nonce or not isinstance(nonce, str) or not hmac.compare_digest(nonce, expected_nonce): + return _BindingRejection( + code="oauth_identity_binding_failed", + description="id_token nonce does not match the authenticated authorization transaction", + ) upstream: Final = _upstream_principal(claims, binding) if isinstance(upstream, _BindingRejection): return upstream @@ -267,7 +363,7 @@ async def _evaluate_binding( code="oauth_principal_mismatch", description="The browser account does not match the selected credential owner.", ) - return None + return _binding_proof(binding, litellm_user_id, server_id, caller) async def enforce_oauth_identity_binding( @@ -279,16 +375,17 @@ async def enforce_oauth_identity_binding( jwks_fetcher: JwksFetcher = _fetch_issuer_jwks, caller_principal_loader: CallerPrincipalLoader = _load_caller_principal, stored_refresh_token_loader: StoredRefreshTokenLoader = _load_stored_refresh_token, -) -> None: + expected_nonce: str | None = None, +) -> str | None: """Validate the exchanged token's upstream principal against the LiteLLM caller. No-op when the server has no binding or it is disabled. In enforce mode a failure raises 403 before the caller returns, stores, or caches the token; in audit mode failures are logged only. A refresh_token grant without an id_token is allowed only when the presented refresh token matches - the caller's stored credential or a sealed bridge envelope already proved ownership. + the caller's stored credential and that credential was previously identity-validated. """ binding: Final = server.oauth_identity_binding - if binding is None or binding.mode == "disabled": + if binding is None or binding.mode not in ("audit", "enforce"): return rejection: Final = await _evaluate_binding( binding=binding, @@ -300,9 +397,10 @@ async def enforce_oauth_identity_binding( jwks_fetcher=jwks_fetcher, caller_principal_loader=caller_principal_loader, stored_refresh_token_loader=stored_refresh_token_loader, + expected_nonce=expected_nonce, ) - if rejection is None: - return + if isinstance(rejection, str): + return rejection if binding.mode == "enforce" else None if binding.mode == "audit": verbose_logger.warning( "oauth_identity_binding audit: server=%s user=%s grant=%s rejected=%s (%s)", diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py index 92bd30694af..824459fd9b9 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py @@ -18,6 +18,11 @@ from litellm._logging import verbose_logger from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( TokenEndpointAuthConfigError, ) +from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( + BindingValidator, + RefreshTokenPresented, + enforce_oauth_identity_binding, +) from litellm.proxy._experimental.mcp_server.oauth_utils import build_upstream_oauth2_token_request from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, @@ -39,6 +44,7 @@ class CredentialPersist(Protocol): refresh_token: str | None, expires_in: int | None, scopes: tuple[str, ...] | None, + identity_binding_proof: str | None = None, ) -> None: ... @@ -78,11 +84,13 @@ class AuthorizationCodeRefresher: persist: CredentialPersist, *, clock: Callable[[], float] = time.time, + identity_validator: BindingValidator = enforce_oauth_identity_binding, ) -> None: self._server_lookup = server_lookup self._token_endpoint = token_endpoint self._persist = persist self._clock = clock + self._identity_validator = identity_validator async def refresh(self, user_id: str, server_id: str, token: OAuthToken) -> OAuthToken | None: if token.refresh_token is None: @@ -104,6 +112,15 @@ class AuthorizationCodeRefresher: except TokenEndpointAuthConfigError as exc: verbose_logger.warning("MCP OAuth refresh misconfigured for server %s: %s", server_id, exc) return None + binding: Final = server.oauth_identity_binding + if binding is not None and binding.mode == "enforce": + await self._identity_validator( + server=server, + token_response={}, + litellm_user_id=user_id, + grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented(token.refresh_token), + ) form: Final = { "grant_type": "refresh_token", "refresh_token": token.refresh_token, @@ -116,12 +133,30 @@ class AuthorizationCodeRefresher: if not isinstance(access_token, str) or not access_token: return None + binding_proof: Final = await self._identity_validator( + server=server, + token_response=body, + litellm_user_id=user_id, + grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented(token.refresh_token), + ) rotated: Final = body.get("refresh_token") new_refresh: Final = rotated if isinstance(rotated, str) and rotated else token.refresh_token expires_in: Final = _parse_expires_in(body.get("expires_in")) scopes: Final = _parse_scopes(body.get("scope")) or token.scopes - await self._persist(user_id, server_id, access_token, new_refresh, expires_in, scopes or None) + if binding_proof is not None: + await self._persist( + user_id, + server_id, + access_token, + new_refresh, + expires_in, + scopes or None, + identity_binding_proof=binding_proof, + ) + else: + await self._persist(user_id, server_id, access_token, new_refresh, expires_in, scopes or None) return OAuthToken( access_token=access_token, expires_at=self._clock() + expires_in if expires_in is not None else None, diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py index 5e28396dcfb..efcc613c539 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py @@ -15,6 +15,7 @@ from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Final from litellm._logging import verbose_logger +from litellm.proxy._experimental.mcp_server.oauth_identity_binding import credential_binding_matches from litellm.proxy._experimental.mcp_server.outbound_credentials.authz_code_refresher import ( AuthorizationCodeRefresher, ) @@ -38,6 +39,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.token_cache_cod OAuthTokenCacheCodec, ) from litellm.proxy._experimental.mcp_server.outbound_credentials.v2_token_store import ( + CredentialReader, V2PerUserTokenStore, ) @@ -69,6 +71,7 @@ async def _persist_credential( refresh_token: str | None, expires_in: int | None, scopes: tuple[str, ...] | None, + identity_binding_proof: str | None = None, ) -> None: from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 store_user_oauth_credential, @@ -86,6 +89,7 @@ async def _persist_credential( expires_in=expires_in, scopes=list(scopes) if scopes else None, skip_byok_guard=True, + identity_binding_proof=identity_binding_proof, ) @@ -172,8 +176,10 @@ class LazyPerUserOAuthTokenStore: *, store_builder: StoreBuilder = _build_per_user_oauth_token_store, redis_available: Callable[[], bool] = _redis_cache_is_available, + credential_reader: CredentialReader = _read_credential, ) -> None: self._server_lookup = server_lookup + self._credential_reader = credential_reader self._store_builder = store_builder self._redis_available = redis_available self._store: InvalidatableOAuthTokenStore | None = None @@ -182,6 +188,13 @@ class LazyPerUserOAuthTokenStore: self._local_fetches = 0 async def fetch(self, user_id: str, server_id: str) -> OAuthToken | None: + server: Final = self._server_lookup(server_id) + binding: Final = server.oauth_identity_binding if server else None + if binding is not None and binding.mode == "enforce": + credential: Final = await self._credential_reader(user_id, server_id) + await self.invalidate(user_id, server_id) + if credential is None or not await credential_binding_matches(binding, user_id, server_id, credential): + return None if self._uses_redis: store = self._store if store is not None: diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 117d95a09de..f6b5441a626 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -1,7 +1,8 @@ from datetime import datetime from typing import Any, Final, Literal -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from typing_extensions import Self from litellm.types.mcp import ( DEFAULT_SUBJECT_TOKEN_TYPE, @@ -42,7 +43,7 @@ class MCPOAuthIdentityBinding(BaseModel): """Per-server policy binding stored per-user OAuth credentials to the authenticated LiteLLM caller. When enabled for an interactive oauth2 server, the token relay validates the upstream OIDC - ``id_token`` (signature via the pinned issuer's JWKS, issuer, audience, expiry) and compares its + ``id_token`` (signature via the pinned issuer's JWKS, issuer, audience, expiry, nonce) and compares its principal claim to the LiteLLM caller's trusted identity before the token is returned, stored, or cached. ``audit`` logs mismatches without changing behavior; ``enforce`` fails closed with 403 ``oauth_principal_mismatch`` and disables the direct ``oauth-user-credential`` POST, which @@ -246,6 +247,14 @@ class MCPServer(BaseModel): """ return self.oauth2_flow == "client_credentials" + @model_validator(mode="after") + def validate_identity_binding_mode(self) -> Self: + binding: Final = self.oauth_identity_binding + if binding is not None and binding.mode != "disabled": + if not self.needs_user_oauth_token or self.delegate_auth_to_upstream: + raise ValueError("oauth_identity_binding requires gateway-managed per-user OAuth2 credentials") + return self + @property def needs_user_oauth_token(self) -> bool: """True if this is an OAuth2 server that relies on per-user tokens (no client_credentials).""" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py index bb2f2ff8b02..16707f6ace8 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py @@ -22,6 +22,7 @@ class _Server: server_id="srv", configured_token_url=None, ): + self.oauth_identity_binding = None self.token_url = token_url self.configured_token_url = configured_token_url self.client_id = client_id @@ -287,3 +288,36 @@ async def test_refresh_uses_admin_entered_token_url_when_issuer_yield_empties_re assert token is not None assert token.access_token == "new-at" assert posted[0][0] == "https://idp.example.com/token" + + +@pytest.mark.asyncio +async def test_identity_rejection_never_persists_or_returns_refreshed_token(): + from unittest.mock import AsyncMock + from fastapi import HTTPException + + validator = AsyncMock(side_effect=HTTPException(status_code=403, detail="oauth_principal_mismatch")) + persist = AsyncMock() + refresher = AuthorizationCodeRefresher( + _lookup(_Server()), _endpoint({"access_token": "foreign-token", "id_token": "foreign-identity"}), + persist, identity_validator=validator, + ) + with pytest.raises(HTTPException) as error: + await refresher.refresh("alice", "srv", OAuthToken(access_token="old", refresh_token="old-rt")) + assert error.value.status_code == 403 + persist.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_verified_refresh_preserves_binding_proof_in_storage(): + from unittest.mock import AsyncMock + + validator = AsyncMock(return_value="verified-binding") + persist = AsyncMock() + refresher = AuthorizationCodeRefresher( + _lookup(_Server()), _endpoint({"access_token": "new", "refresh_token": "rotated"}), + persist, identity_validator=validator, + ) + token = await refresher.refresh("alice", "srv", OAuthToken(access_token="old", refresh_token="old-rt")) + assert token.access_token == "new" + assert token.refresh_token == "rotated" + assert persist.await_args.kwargs["identity_binding_proof"] == "verified-binding" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py index ca32cf2bb8d..bd190b09c82 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py @@ -246,3 +246,28 @@ async def test_lazy_store_invalidate_works_after_redis_chain_is_built() -> None: assert build_calls == 1 assert redis_store.invalidations == [("u", "s")] + + +@pytest.mark.asyncio +async def test_enforcement_invalidates_cached_legacy_credentials_before_use(): + from litellm.types.mcp import MCPAuth, MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + + server = MCPServer( + server_id="srv", name="srv", transport=MCPTransport.http, auth_type=MCPAuth.oauth2, + oauth_identity_binding=MCPOAuthIdentityBinding( + mode="enforce", issuer="https://idp.example.com", audiences=["client"], + ), + ) + cached = _RecordingStore("belongs-to-bob") + + async def read_legacy(user_id, server_id): + return {"access_token": "belongs-to-bob", "refresh_token": "bobs-refresh"} + + store = LazyPerUserOAuthTokenStore( + lambda server_id: server, store_builder=lambda lookup: (cached, False), + redis_available=lambda: False, credential_reader=read_legacy, + ) + assert await store.fetch("alice", "srv") is None + assert cached.calls == [] + assert cached.invalidations == [("alice", "srv")] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py index 355b3bfd30e..8e2adbeb06e 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py @@ -628,6 +628,7 @@ async def test_oauth_round_trip_returns_payload(): access_token, refresh_token="rfr-xyz", scopes=["a", "b"], + identity_binding_proof="verified-proof", ) stored = _stored_value(prisma) @@ -642,6 +643,7 @@ async def test_oauth_round_trip_returns_payload(): assert result["access_token"] == access_token assert result["refresh_token"] == "rfr-xyz" assert result["scopes"] == ["a", "b"] + assert result["identity_binding_proof"] == "verified-proof" @pytest.mark.asyncio @@ -1184,6 +1186,7 @@ class _RefreshResponse: def _refresh_server(**overrides): base = dict( + oauth_identity_binding=None, token_url="https://idp.example.com/token", server_id="srv-1", client_id="cid", @@ -1309,7 +1312,7 @@ async def test_refresh_user_oauth_token_uses_client_secret_basic(monkeypatch): sends HTTP Basic and keeps the secret out of the body.""" import litellm.proxy._experimental.mcp_server.db as db_mod - server = MagicMock() + server = MagicMock(oauth_identity_binding=None) server.token_url = "https://idp.example.com/oauth2/token" server.server_id = "srv" server.client_id = "cid" @@ -1348,7 +1351,7 @@ async def test_refresh_user_oauth_token_defaults_to_client_secret_post(monkeypat the body (client_secret_post) and sends no Authorization header.""" import litellm.proxy._experimental.mcp_server.db as db_mod - server = MagicMock() + server = MagicMock(oauth_identity_binding=None) server.token_url = "https://idp.example.com/oauth2/token" server.server_id = "srv" server.client_id = "cid" @@ -1609,3 +1612,20 @@ def test_partial_update_defers_omitted_eligibility_fields_to_the_stored_row(): request = UpdateMCPServerRequest(server_id="relay-update", per_server_oauth_discovery=True) assert request.per_server_oauth_discovery is True + + +@pytest.mark.asyncio +async def test_enforcement_rejects_preexisting_unverified_credential(): + from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + + server = MCPServer( + server_id="srv-1", name="srv-1", url="https://mcp.example.com", transport=MCPTransport.http, auth_type=MCPAuth.oauth2, + oauth_identity_binding=MCPOAuthIdentityBinding( + mode="enforce", issuer="https://idp.example.com", audiences=["client"], + ), + ) + result = await resolve_valid_user_oauth_token( + user_id="alice", server=server, + cred={"access_token": "belongs-to-bob", "refresh_token": "bobs-refresh-token"}, + ) + assert result is None diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 590d6606154..42187cb1771 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -8194,15 +8194,17 @@ async def test_token_exchange_refresh_passes_presented_refresh_ownership(): @pytest.mark.asyncio -async def test_token_exchange_authorization_code_passes_no_refresh_ownership(): +async def test_token_exchange_authorization_code_passes_no_refresh_ownership(monkeypatch): from fastapi import Request from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( exchange_token_with_server, + seal_bridge_authorization_code, ) from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + monkeypatch.setenv("LITELLM_SALT_KEY", "identity-binding-test-salt") server = MCPServer( server_id="srv-1", name="srv-1", @@ -8244,16 +8246,17 @@ async def test_token_exchange_authorization_code_passes_no_refresh_ownership(): request=request, mcp_server=server, grant_type="authorization_code", - code="auth-code", + code=seal_bridge_authorization_code("auth-code", "user-a", "srv-1", "login-nonce"), redirect_uri="https://litellm.example.com/callback", client_id="cid", client_secret=None, - code_verifier=None, + code_verifier="test-verifier", ) assert result.status_code == 200 assert json.loads(result.body)["access_token"] == "at" assert enforce.await_args.kwargs["refresh_ownership"] is None + assert enforce.await_args.kwargs["expected_nonce"] == "login-nonce" def _upstream_token_response(status_code: int, *, json_body: object = None, text_body: str = "") -> "httpx.Response": @@ -11049,3 +11052,54 @@ def test_introspect_route_answers_for_authenticated_caller(monkeypatch): assert active.status_code == 200 assert active.json()["active"] is True assert active.json()["sub"] == "u1" + + +@pytest.mark.asyncio +async def test_identity_bound_authorization_carries_nonce_and_caller_through_callback(monkeypatch): + from http.cookies import SimpleCookie + from urllib.parse import parse_qs, urlparse + from fastapi import Request + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + _oauth_state_cookie_name, authorize_with_server, callback, open_bridge_authorization_code, + ) + from litellm.types.mcp import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + + monkeypatch.setenv("LITELLM_SALT_KEY", "identity-binding-test-salt") + server = MCPServer( + server_id="srv", name="srv", transport=MCPTransport.http, auth_type=MCPAuth.oauth2, + client_id="client", authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", + oauth_identity_binding=MCPOAuthIdentityBinding( + mode="enforce", issuer="https://idp.example.com", audiences=["client"], + ), + ) + request = Request({"type": "http", "scheme": "https", "server": ("proxy.example.com", 443), + "path": "/authorize", "query_string": b"", "headers": []}) + with ( + patch( # test-quality-ok: isolate authenticated request resolution from the real encrypted OAuth round trip + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._extract_user_id_from_request", + new=AsyncMock(return_value="alice")), + patch( # test-quality-ok: isolate user access lookup while testing nonce and caller preservation + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._bridge_authorize_access_denial", + new=AsyncMock(return_value=None)), + ): + authorized = await authorize_with_server( + request, server, "client", "http://127.0.0.1:6274/callback", state="client-state", + code_challenge="pkce-challenge", code_challenge_method="S256", + ) + query = parse_qs(urlparse(authorized.headers["location"]).query) + assert len(query["nonce"][0]) >= 32 + cookies = SimpleCookie() + cookies.load(authorized.headers["set-cookie"]) + name = _oauth_state_cookie_name(query["state"][0]) + callback_request = Request({**request.scope, "path": "/callback", + "headers": [(b"cookie", f"{name}={cookies[name].value}".encode())]}) + completed = await callback(callback_request, code="upstream-code", state=query["state"][0]) + returned = parse_qs(urlparse(completed.headers["location"]).query) + sealed = open_bridge_authorization_code(returned["code"][0]) + assert sealed.litellm_user_id == "alice" + assert sealed.mcp_server_id == "srv" + assert sealed.upstream_code == "upstream-code" + assert sealed.oauth_nonce == query["nonce"][0] + assert returned["state"] == ["client-state"] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py index 36364c32ae5..6f291865e20 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py @@ -13,7 +13,9 @@ from pydantic import ValidationError from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( RefreshOwnershipProven, + VerifiedRefreshToken, RefreshTokenPresented, + current_binding_proof, _discover_jwks_url, _fetch_issuer_jwks, _load_caller_principal, @@ -21,7 +23,7 @@ from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( _select_signing_key, enforce_oauth_identity_binding, ) -from litellm.types.mcp import MCPTransport +from litellm.types.mcp import MCPAuth, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer ISSUER: Final = "https://idp.example.com" @@ -48,6 +50,8 @@ def _sign_id_token(claims: Mapping[str, object]) -> str: "aud": AUDIENCE, "exp": int(time.time()) + 300, "iat": int(time.time()), + "nonce": "test-nonce", + "sub": "upstream-user", **claims, } return jwt.encode(payload, _PRIVATE_PEM, algorithm="RS256", headers={"kid": KID}) @@ -65,8 +69,8 @@ def _caller_loader(email: str | None): def _stored_refresh_token_loader(refresh_token: str | None): - async def load(_user_id: str, _server_id: str) -> str | None: - return refresh_token + async def load(_user_id: str, _server_id: str, _binding: MCPOAuthIdentityBinding) -> VerifiedRefreshToken | None: + return VerifiedRefreshToken(refresh_token, "verified-binding") if refresh_token else None return load @@ -77,6 +81,7 @@ def _server(mode: str = "enforce", **binding_overrides: object) -> MCPServer: name="srv-1", url="https://mcp.example.com", transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, oauth_identity_binding=MCPOAuthIdentityBinding( mode=mode, issuer=ISSUER, @@ -181,7 +186,9 @@ async def test_load_caller_principal_supports_user_id_and_database_email(): @pytest.mark.asyncio async def test_load_stored_refresh_token_returns_credential_and_fails_closed(): - get_credential: Final = AsyncMock(return_value={"refresh_token": "rt-1"}) + binding: Final = _server(caller_field="user_id", principal_claim="sub").oauth_identity_binding + proof: Final = await current_binding_proof(binding, "user-a", "srv-1") + get_credential: Final = AsyncMock(return_value={"refresh_token": "rt-1", "identity_binding_proof": proof}) with ( patch( # test-quality-ok: stored-token loading is a lazy database boundary without injection "litellm.proxy._experimental.mcp_server.db.get_user_oauth_credential", @@ -192,13 +199,15 @@ async def test_load_stored_refresh_token_returns_credential_and_fails_closed(): return_value="prisma", ), ): - assert await _load_stored_refresh_token("user-a", "srv-1") == "rt-1" + assert await _load_stored_refresh_token("user-a", "srv-1", binding) == VerifiedRefreshToken("rt-1", proof) + get_credential.return_value = {"refresh_token": "rt-1"} + assert await _load_stored_refresh_token("user-a", "srv-1", binding) is None with patch( # test-quality-ok: stored-token loading is a lazy database boundary without injection "litellm.proxy.utils.get_prisma_client_or_throw", side_effect=RuntimeError("database unavailable"), ): - assert await _load_stored_refresh_token("user-a", "srv-1") is None + assert await _load_stored_refresh_token("user-a", "srv-1", binding) is None @pytest.mark.asyncio @@ -209,11 +218,12 @@ async def test_matching_principal_passes(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) - assert result is None + assert result is not None @pytest.mark.asyncio @@ -225,6 +235,7 @@ async def test_mismatched_principal_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), @@ -243,6 +254,7 @@ async def test_missing_upstream_principal_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), @@ -263,6 +275,7 @@ async def test_jwks_fetch_failure_is_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=fail, caller_principal_loader=_caller_loader("alice@example.com"), @@ -284,6 +297,7 @@ async def test_missing_signing_key_is_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=no_keys, caller_principal_loader=_caller_loader("alice@example.com"), @@ -301,6 +315,7 @@ async def test_missing_caller_principal_is_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader(None), @@ -334,11 +349,12 @@ async def test_user_id_principal_matching_uses_exact_comparison(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("user-a"), ) - assert result is None + assert result is not None @pytest.mark.asyncio @@ -349,6 +365,7 @@ async def test_missing_id_token_rejected_on_authorization_code(): token_response={"access_token": "at"}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), @@ -369,7 +386,7 @@ async def test_refresh_without_id_token_allowed_when_presented_token_matches_sto caller_principal_loader=_caller_loader("alice@example.com"), stored_refresh_token_loader=_stored_refresh_token_loader("rt-1"), ) - assert result is None + assert result is not None @pytest.mark.asyncio @@ -407,21 +424,18 @@ async def test_refresh_without_id_token_rejects_missing_stored_token(): @pytest.mark.asyncio -async def test_refresh_without_id_token_passes_when_bridge_proves_ownership(): - async def fail_if_called(_user_id: str, _server_id: str) -> str | None: - raise AssertionError("stored refresh token loader should not be called") - - result: Final = await enforce_oauth_identity_binding( - server=_server(), - token_response={"access_token": "at"}, - litellm_user_id="user-a", - grant_type="refresh_token", - refresh_ownership=RefreshOwnershipProven(), - jwks_fetcher=_jwks_fetcher, - caller_principal_loader=_caller_loader("alice@example.com"), - stored_refresh_token_loader=fail_if_called, - ) - assert result is None +async def test_identity_envelope_does_not_prove_upstream_binding(): + with pytest.raises(HTTPException) as error: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id="user-a", + grant_type="refresh_token", + refresh_ownership=RefreshOwnershipProven(), + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert error.value.status_code == 403 @pytest.mark.asyncio @@ -464,6 +478,7 @@ async def test_audit_mode_logs_but_does_not_reject(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), @@ -480,6 +495,7 @@ async def test_unverified_email_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), @@ -503,6 +519,7 @@ async def test_wrong_issuer_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), @@ -519,6 +536,7 @@ async def test_no_litellm_identity_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id=None, grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), @@ -533,6 +551,7 @@ async def test_disabled_binding_is_noop(): token_response={"access_token": "at"}, litellm_user_id=None, grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader(None), @@ -553,6 +572,7 @@ async def test_no_binding_is_noop(): token_response={"access_token": "at"}, litellm_user_id=None, grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader(None), @@ -576,9 +596,52 @@ async def test_wrong_audience_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) assert exc_info.value.status_code == 403 assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("nonce", [None, "another-login"]) +async def test_authorization_code_rejects_missing_or_foreign_nonce(nonce): + token = _sign_id_token({"email": "alice@example.com", "email_verified": True, "nonce": nonce}) + with pytest.raises(HTTPException) as error: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + expected_nonce="this-login", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert error.value.status_code == 403 + assert error.value.detail["error"] == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +async def test_binding_proof_rejects_changed_user_or_policy(): + from litellm.proxy._experimental.mcp_server.oauth_identity_binding import credential_binding_matches + + binding = _server(caller_field="user_id", principal_claim="sub").oauth_identity_binding + proof = await current_binding_proof(binding, "alice", "srv-1") + credential = {"identity_binding_proof": proof} + assert await credential_binding_matches(binding, "alice", "srv-1", credential) + assert not await credential_binding_matches(binding, "bob", "srv-1", credential) + assert not await credential_binding_matches(binding, "alice", "other-server", credential) + changed = binding.model_copy(update={"audiences": ["different-client"]}) + assert not await credential_binding_matches(changed, "alice", "srv-1", credential) + + +@pytest.mark.parametrize("auth_type", [MCPAuth.oauth_delegate, MCPAuth.true_passthrough]) +def test_identity_binding_rejects_modes_without_gateway_credential_custody(auth_type): + with pytest.raises(ValidationError, match="gateway-managed per-user"): + MCPServer( + server_id="srv", name="srv", transport=MCPTransport.http, auth_type=auth_type, + oauth_identity_binding=_server().oauth_identity_binding, + ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index 6a20688c09d..5e00e7d75be 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -4807,7 +4807,7 @@ async def test_store_mcp_oauth_user_credential_blocked_when_identity_binding_enf """The direct opaque-token POST must be closed for enforce-mode identity-bound servers, otherwise it bypasses the token-relay principal check.""" from litellm.proxy._types import MCPOAuthUserCredentialRequest - from litellm.types.mcp import MCPTransport + from litellm.types.mcp import MCPAuth, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer if not mgmt_endpoints.MCP_AVAILABLE: @@ -4824,6 +4824,7 @@ async def test_store_mcp_oauth_user_credential_blocked_when_identity_binding_enf name=server_id, url="https://mcp.example.com", transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, oauth_identity_binding=MCPOAuthIdentityBinding( mode="enforce", issuer="https://idp.example.com", From 1273e46b8bf4821643eb4b948517c95de34bf632 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 26 Aug 2026 15:45:25 -0400 Subject: [PATCH 029/126] feat(redis): add ElastiCache IAM authentication --- litellm/_redis.py | 57 ++++- litellm/_redis_credential_provider.py | 85 +++++++- litellm/proxy/_types.py | 4 + .../cache_settings_endpoints.py | 36 ++++ .../coordination_redis_endpoints.py | 29 +++ tests/test_litellm/test_redis.py | 196 +++++++++++++++++- .../test_redis_credential_provider.py | 122 +++++++++++ 7 files changed, 520 insertions(+), 9 deletions(-) create mode 100644 tests/test_litellm/test_redis_credential_provider.py diff --git a/litellm/_redis.py b/litellm/_redis.py index 3e68d50cf16..e289cc3b6c3 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -24,6 +24,7 @@ from redis.credentials import CredentialProvider from litellm import get_secret, get_secret_str from litellm._redis_credential_provider import ( AzureADCredentialProvider, + ElastiCacheIAMCredentialProvider, GCPIAMCredentialProvider, _generate_gcp_iam_access_token, ) @@ -75,6 +76,10 @@ def _get_redis_kwargs(): "azure_client_id", "azure_tenant_id", "azure_client_secret", + "aws_iam_auth", + "aws_iam_user_name", + "aws_iam_cache_name", + "aws_iam_region", } available_args: Final = {x for x in _unwrapped_init_args(redis.Redis) if x not in exclude_args} | include_args @@ -270,6 +275,32 @@ def _redis_kwargs_from_environment(): return return_dict +def _is_true(value: object | None) -> bool: + return value is True or (isinstance(value, str) and value.lower() == "true") + + +def _build_elasticache_iam_provider(redis_kwargs: dict) -> ElastiCacheIAMCredentialProvider | None: + if not _is_true(redis_kwargs.get("aws_iam_auth")): + return None + + required_settings: Final = { + "aws_iam_user_name": redis_kwargs.get("aws_iam_user_name"), + "aws_iam_cache_name": redis_kwargs.get("aws_iam_cache_name"), + "aws_iam_region": redis_kwargs.get("aws_iam_region") + or get_secret_str("AWS_REGION") + or get_secret_str("AWS_DEFAULT_REGION"), + } + missing_settings: Final = tuple(name for name, value in required_settings.items() if not value) + if missing_settings: + raise ValueError("AWS ElastiCache IAM Redis authentication requires: " + ", ".join(missing_settings)) + + return ElastiCacheIAMCredentialProvider( + user_name=str(required_settings["aws_iam_user_name"]), + cache_name=str(required_settings["aws_iam_cache_name"]), + region=str(required_settings["aws_iam_region"]), + ) + + def create_gcp_iam_redis_connect_func( service_account: str, ssl_ca_certs: str | None = None, @@ -540,6 +571,7 @@ def _get_redis_client_logic(**env_overrides): _azure_redis_ad_token: Final = redis_kwargs.get("azure_redis_ad_token") or get_secret("REDIS_AZURE_AD_TOKEN") _azure_ad_enabled: Final = _azure_redis_ad_token is not None and str(_azure_redis_ad_token).lower() == "true" + _aws_iam_enabled: Final = _is_true(redis_kwargs.get("aws_iam_auth")) if _azure_ad_enabled and _gcp_service_account is not None: verbose_logger.warning( @@ -560,13 +592,24 @@ def _get_redis_client_logic(**env_overrides): azure_tenant_id=_azure_tenant_id, azure_client_secret=_azure_client_secret, ) - # Marker for async paths to detect Azure AD auth. The live credential - # object is attached separately as `_azure_credential` by - # `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret - # are intentionally NOT exposed on the function to avoid leaking - # credentials via inspection or logging. redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True + if _aws_iam_enabled and _gcp_service_account is not None: + verbose_logger.warning( + "Both GCP IAM (gcp_service_account) and AWS ElastiCache IAM (aws_iam_auth) are configured " + "for Redis. Using GCP IAM. Remove one to avoid misconfiguration." + ) + elif _aws_iam_enabled and _azure_ad_enabled: + verbose_logger.warning( + "Both Azure AD (azure_redis_ad_token) and AWS ElastiCache IAM (aws_iam_auth) are configured " + "for Redis. Using Azure AD. Remove one to avoid misconfiguration." + ) + elif _aws_iam_enabled: + aws_provider: Final = _build_elasticache_iam_provider(redis_kwargs) + if aws_provider is not None: + verbose_logger.debug("Setting up AWS ElastiCache IAM authentication for Redis.") + redis_kwargs["credential_provider"] = aws_provider + redis_kwargs.pop("gcp_service_account", None) redis_kwargs.pop("gcp_ssl_ca_certs", None) @@ -575,6 +618,10 @@ def _get_redis_client_logic(**env_overrides): redis_kwargs.pop("azure_client_id", None) redis_kwargs.pop("azure_tenant_id", None) redis_kwargs.pop("azure_client_secret", None) + redis_kwargs.pop("aws_iam_auth", None) + redis_kwargs.pop("aws_iam_user_name", None) + redis_kwargs.pop("aws_iam_cache_name", None) + redis_kwargs.pop("aws_iam_region", None) if redis_kwargs.get("credential_provider") is not None: redis_kwargs.pop("redis_connect_func", None) diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index ba0398789a6..a20bbfc530c 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -1,7 +1,8 @@ import asyncio import threading import time -from typing import Final, Protocol +from typing import Any, Final, Protocol +from urllib.parse import quote from redis.credentials import CredentialProvider @@ -117,6 +118,88 @@ class GCPIAMCredentialProvider(CredentialProvider): return (token,) +_ELASTICACHE_SERVICE_NAME: Final = "elasticache" +_ELASTICACHE_TOKEN_TTL_SECONDS: Final = 900 + + +class _FrozenBotocoreCredentials(Protocol): + access_key: str + secret_key: str + token: str | None + + +class _BotocoreCredentials(Protocol): + def get_frozen_credentials(self) -> _FrozenBotocoreCredentials: ... + + +class _BotocoreCredentialsResolver(Protocol): + def __call__(self) -> _BotocoreCredentials | None: ... + + +class ElastiCacheIAMCredentialProvider(CredentialProvider): + def __init__( + self, + user_name: str, + cache_name: str, + region: str, + credentials_resolver: _BotocoreCredentialsResolver | None = None, + token_lifetime_seconds: int = _ELASTICACHE_TOKEN_TTL_SECONDS, + ) -> None: + self._user_name = user_name + self._cache_name = cache_name + self._region = region + self._credentials_resolver = credentials_resolver or self._resolve_credentials + self._credentials: _BotocoreCredentials | None = None + self._token_lifetime_seconds = token_lifetime_seconds + + @staticmethod + def _resolve_credentials() -> Any: + try: + import botocore.session + except ImportError as e: + raise ImportError( + "botocore is required for ElastiCache IAM Redis authentication. Install it with: pip install boto3" + ) from e + + return botocore.session.get_session().get_credentials() + + def _get_credentials(self) -> tuple[str, str]: + credentials: Final = self._credentials if self._credentials is not None else self._credentials_resolver() + if credentials is None: + raise RuntimeError("Unable to resolve AWS credentials for ElastiCache IAM Redis authentication") + self._credentials = credentials + + frozen_credentials: Final = credentials.get_frozen_credentials() + if frozen_credentials is None: + raise RuntimeError("Unable to resolve AWS credentials for ElastiCache IAM Redis authentication") + + try: + from botocore.auth import SigV4QueryAuth + from botocore.awsrequest import AWSRequest + except ImportError as e: + raise ImportError( + "botocore is required for ElastiCache IAM Redis authentication. Install it with: pip install boto3" + ) from e + + request: Final = AWSRequest( + method="GET", + url=(f"https://{self._cache_name}/?Action=connect&User={quote(self._user_name, safe='')}"), + ) + SigV4QueryAuth( + frozen_credentials, + _ELASTICACHE_SERVICE_NAME, + self._region, + expires=self._token_lifetime_seconds, + ).add_auth(request) + return self._user_name, request.url.removeprefix("https://") + + def get_credentials(self) -> tuple[str, str]: + return self._get_credentials() + + async def get_credentials_async(self) -> tuple[str, str]: + return await asyncio.to_thread(self._get_credentials) + + class AzureADCredentialProvider(CredentialProvider): """ redis.credentials.CredentialProvider implementation that supplies Azure AD diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c22bb76629d..99103ff3706 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2427,6 +2427,10 @@ class CoordinationRedisParams(LiteLLMPydanticObjectBase): ) sentinel_password: str | None = Field(None, description="password for the sentinel nodes") service_name: str | None = Field(None, description="sentinel service name") + aws_iam_auth: bool | str | None = Field(None, description="enable AWS ElastiCache IAM authentication") + aws_iam_user_name: str | None = Field(None, description="AWS ElastiCache IAM user name") + aws_iam_cache_name: str | None = Field(None, description="AWS ElastiCache cache name") + aws_iam_region: str | None = Field(None, description="AWS region for ElastiCache IAM authentication") def has_connection_target(self) -> bool: return any(value is not None for value in (self.host, self.url, self.startup_nodes, self.sentinel_nodes)) diff --git a/litellm/types/management_endpoints/cache_settings_endpoints.py b/litellm/types/management_endpoints/cache_settings_endpoints.py index 32b32991449..10aece6efa4 100644 --- a/litellm/types/management_endpoints/cache_settings_endpoints.py +++ b/litellm/types/management_endpoints/cache_settings_endpoints.py @@ -237,4 +237,40 @@ CACHE_SETTINGS_FIELDS: Final[list[CacheSettingsField]] = [ ui_field_name="SSL Check Hostname", redis_type=None, ), + CacheSettingsField( + field_name="aws_iam_auth", + field_type="Boolean", + field_value=None, + field_description="Enable AWS ElastiCache IAM authentication", + field_default=False, + ui_field_name="AWS IAM Authentication", + redis_type=None, + ), + CacheSettingsField( + field_name="aws_iam_user_name", + field_type="String", + field_value=None, + field_description="AWS ElastiCache IAM user name", + field_default=None, + ui_field_name="AWS IAM User Name", + redis_type=None, + ), + CacheSettingsField( + field_name="aws_iam_cache_name", + field_type="String", + field_value=None, + field_description="AWS ElastiCache cache name", + field_default=None, + ui_field_name="AWS IAM Cache Name", + redis_type=None, + ), + CacheSettingsField( + field_name="aws_iam_region", + field_type="String", + field_value=None, + field_description="AWS region for ElastiCache IAM authentication", + field_default=None, + ui_field_name="AWS IAM Region", + redis_type=None, + ), ] diff --git a/litellm/types/management_endpoints/coordination_redis_endpoints.py b/litellm/types/management_endpoints/coordination_redis_endpoints.py index 30033346ed7..86b32752cbb 100644 --- a/litellm/types/management_endpoints/coordination_redis_endpoints.py +++ b/litellm/types/management_endpoints/coordination_redis_endpoints.py @@ -102,4 +102,33 @@ COORDINATION_REDIS_SETTINGS_FIELDS: Final[list[CoordinationRedisSettingsField]] ui_field_name="Service Name", section="sentinel", ), + CoordinationRedisSettingsField( + field_name="aws_iam_auth", + field_type="Boolean", + field_description="Enable AWS ElastiCache IAM authentication", + field_default=False, + ui_field_name="AWS IAM Authentication", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="aws_iam_user_name", + field_type="String", + field_description="AWS ElastiCache IAM user name", + ui_field_name="AWS IAM User Name", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="aws_iam_cache_name", + field_type="String", + field_description="AWS ElastiCache cache name", + ui_field_name="AWS IAM Cache Name", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="aws_iam_region", + field_type="String", + field_description="AWS region for ElastiCache IAM authentication", + ui_field_name="AWS IAM Region", + section="connection", + ), ] diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index a96e8541e06..c99a3a074fb 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -24,6 +24,7 @@ from litellm._redis import ( ) from litellm._redis_credential_provider import ( AzureADCredentialProvider, + ElastiCacheIAMCredentialProvider, GCPIAMCredentialProvider, _token_cache, ) @@ -78,6 +79,8 @@ def clean_redis_environment(monkeypatch): "REDIS_URL", "REDIS_CLUSTER_NODES", "REDIS_SENTINEL_NODES", + "AWS_REGION", + "AWS_DEFAULT_REGION", *_get_redis_env_kwarg_mapping(), ): monkeypatch.delenv(var, raising=False) @@ -110,6 +113,17 @@ def test_credential_provider_is_not_environment_derived(): assert "credential_provider" not in mapping.values() +def test_aws_iam_settings_are_environment_derived(): + allowed = _get_redis_kwargs() + mapping = _get_redis_env_kwarg_mapping() + + assert {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} <= allowed + assert mapping["REDIS_AWS_IAM_AUTH"] == "aws_iam_auth" + assert mapping["REDIS_AWS_IAM_USER_NAME"] == "aws_iam_user_name" + assert mapping["REDIS_AWS_IAM_CACHE_NAME"] == "aws_iam_cache_name" + assert mapping["REDIS_AWS_IAM_REGION"] == "aws_iam_region" + + def test_sync_direct_preserves_credential_provider_identity(clean_redis_environment): provider = _StubCredentialProvider() @@ -300,6 +314,180 @@ def test_gcp_kwargs_never_survive_client_logic(clean_redis_environment, override assert "gcp_ssl_ca_certs" not in redis_kwargs +def test_aws_iam_environment_settings_install_provider(clean_redis_environment, monkeypatch): + monkeypatch.setenv("REDIS_AWS_IAM_AUTH", "true") + monkeypatch.setenv("REDIS_AWS_IAM_USER_NAME", "iam-user") + monkeypatch.setenv("REDIS_AWS_IAM_CACHE_NAME", "cache.example.com") + monkeypatch.setenv("REDIS_AWS_IAM_REGION", "us-east-1") + + redis_kwargs = _get_redis_client_logic(host="cache.example.com", port=6379) + + assert isinstance(redis_kwargs["credential_provider"], ElastiCacheIAMCredentialProvider) + assert not {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} & redis_kwargs.keys() + + +def test_aws_iam_settings_are_removed_for_url_and_static_credentials(clean_redis_environment): + redis_kwargs = _get_redis_client_logic( + url="rediss://url-user:url-pass@cache.example.com:6380", + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + username="static-user", + password="static-password", + ) + + assert isinstance(redis_kwargs["credential_provider"], ElastiCacheIAMCredentialProvider) + assert redis_kwargs["url"] == "rediss://cache.example.com:6380" + assert "username" not in redis_kwargs + assert "password" not in redis_kwargs + assert not {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} & redis_kwargs.keys() + + +@pytest.mark.parametrize("missing", ["aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"]) +def test_aws_iam_missing_setting_fails_closed(clean_redis_environment, missing): + settings = { + "aws_iam_auth": True, + "aws_iam_user_name": "iam-user", + "aws_iam_cache_name": "cache.example.com", + "aws_iam_region": "us-east-1", + } + settings[missing] = None + + with pytest.raises(ValueError, match=missing): + _get_redis_client_logic(host="cache.example.com", port=6379, **settings) + + +@pytest.mark.parametrize("region_var", ["AWS_REGION", "AWS_DEFAULT_REGION"]) +def test_aws_iam_region_falls_back_to_environment(clean_redis_environment, monkeypatch, region_var): + monkeypatch.setenv(region_var, "sa-east-1") + + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + ) + + provider = redis_kwargs["credential_provider"] + assert isinstance(provider, ElastiCacheIAMCredentialProvider) + assert provider._region == "sa-east-1" + + +def test_aws_iam_region_prefers_aws_region_over_default_region(clean_redis_environment, monkeypatch): + monkeypatch.setenv("AWS_REGION", "sa-east-1") + monkeypatch.setenv("AWS_DEFAULT_REGION", "eu-west-1") + + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + ) + + assert redis_kwargs["credential_provider"]._region == "sa-east-1" + + +def test_aws_iam_region_prefers_explicit_over_environment(clean_redis_environment, monkeypatch): + monkeypatch.setenv("AWS_REGION", "sa-east-1") + + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="explicit-region", + ) + + assert redis_kwargs["credential_provider"]._region == "explicit-region" + + +def test_aws_iam_settings_map_to_distinct_provider_fields(clean_redis_environment): + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + aws_iam_auth=True, + aws_iam_user_name="iam-user-value", + aws_iam_cache_name="iam-cache-value", + aws_iam_region="iam-region-value", + ) + + provider = redis_kwargs["credential_provider"] + assert isinstance(provider, ElastiCacheIAMCredentialProvider) + assert provider._user_name == "iam-user-value" + assert provider._cache_name == "iam-cache-value" + assert provider._region == "iam-region-value" + + +@pytest.mark.parametrize("aws_iam_auth", [False, "false"]) +def test_aws_iam_auth_disabled_does_not_install_provider(clean_redis_environment, aws_iam_auth): + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + aws_iam_auth=aws_iam_auth, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) + + assert "credential_provider" not in redis_kwargs + assert not {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} & redis_kwargs.keys() + + +def test_explicit_provider_wins_over_aws_iam(clean_redis_environment): + provider = _StubCredentialProvider() + + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + credential_provider=provider, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) + + assert redis_kwargs["credential_provider"] is provider + assert "aws_iam_auth" not in redis_kwargs + + +def test_gcp_wins_over_aws_iam(clean_redis_environment): + with patch("litellm._redis.create_gcp_iam_redis_connect_func") as mock_gcp: + mock_gcp.return_value = _gcp_marker_callback() + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + gcp_service_account="sa@example.com", + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) + + assert "credential_provider" not in redis_kwargs + assert redis_kwargs["redis_connect_func"] is mock_gcp.return_value + + +def test_azure_wins_over_aws_iam(clean_redis_environment): + with patch("litellm._redis.create_azure_ad_redis_connect_func") as mock_azure: + mock_azure.return_value = MagicMock() + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + azure_redis_ad_token="true", + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) + + assert "credential_provider" not in redis_kwargs + assert redis_kwargs["redis_connect_func"] is mock_azure.return_value + + def test_provider_keeps_the_rest_of_the_url_intact(clean_redis_environment): provider = _StubCredentialProvider() @@ -1530,9 +1718,11 @@ def test_async_sentinel_keeps_the_credential_provider_off_the_monitors(markers, "redis_connect_func": SimpleNamespace(**markers), } - with patch("litellm._redis.async_redis.Sentinel") as mock_sentinel_cls: - with patch("litellm._redis._get_redis_client_logic", return_value=redis_kwargs): - get_redis_async_client() + with ( + patch("litellm._redis.async_redis.Sentinel") as mock_sentinel_cls, + patch("litellm._redis._get_redis_client_logic", return_value=redis_kwargs), + ): + get_redis_async_client() sentinel_kwargs = mock_sentinel_cls.call_args[1]["sentinel_kwargs"] assert sentinel_kwargs["password"] == sentinel_password diff --git a/tests/test_litellm/test_redis_credential_provider.py b/tests/test_litellm/test_redis_credential_provider.py new file mode 100644 index 00000000000..6299d4d42e0 --- /dev/null +++ b/tests/test_litellm/test_redis_credential_provider.py @@ -0,0 +1,122 @@ +import asyncio +from types import SimpleNamespace +from urllib.parse import parse_qs, urlsplit + +import pytest + +from litellm._redis_credential_provider import ( + ElastiCacheIAMCredentialProvider, + _BotocoreCredentials, +) + + +class _FakeCredentials: + def __init__(self, access_key: str) -> None: + self.access_key = access_key + self.secret_key = "synthetic-secret" + self.token = "synthetic-session-token" + + def get_frozen_credentials(self): + return self + + +class _FakeResolver: + def __init__(self, credentials: _BotocoreCredentials | None) -> None: + self.credentials = credentials + self.calls = 0 + + def __call__(self): + self.calls += 1 + return self.credentials + + +class _RotatingFakeCredentials: + def __init__(self) -> None: + self.calls = 0 + + def __bool__(self) -> bool: + return False + + def get_frozen_credentials(self): + self.calls += 1 + return SimpleNamespace( + access_key=f"AKIA-SYNTHETIC-{self.calls}", + secret_key="synthetic-secret", + token="synthetic-session-token", + ) + + +def test_elasticache_provider_signs_expected_query(): + resolver = _FakeResolver(_FakeCredentials("AKIA-SYNTHETIC")) + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + credentials_resolver=resolver, + ) + + user_name, token = provider.get_credentials() + parsed = urlsplit("https://" + token) + query = parse_qs(parsed.query) + + assert user_name == "iam-user" + assert parsed.netloc == "cache.example.com" + assert query["Action"] == ["connect"] + assert query["User"] == ["iam-user"] + assert query["X-Amz-Expires"] == ["900"] + assert "elasticache" in query["X-Amz-Credential"][0] + assert query["X-Amz-Credential"][0].split("/")[2] == "us-east-1" + assert not token.startswith("https://") + + +def test_elasticache_provider_resolves_credentials_once_but_refreshes_signature(): + rotating_credentials = _RotatingFakeCredentials() + resolver = _FakeResolver(rotating_credentials) + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + credentials_resolver=resolver, + ) + + first = provider.get_credentials() + second = provider.get_credentials() + async_result = asyncio.run(provider.get_credentials_async()) + + assert first[0] == second[0] == async_result[0] == "iam-user" + assert first[1] != second[1] + assert async_result[1] != second[1] + assert resolver.calls == 1 + assert rotating_credentials.calls == 3 + + +def test_elasticache_provider_reports_missing_credentials(): + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + credentials_resolver=_FakeResolver(None), + ) + + with pytest.raises(RuntimeError, match="Unable to resolve AWS credentials"): + provider.get_credentials() + + +def test_elasticache_provider_recovers_after_a_failed_resolution(): + resolver = _FakeResolver(None) + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + credentials_resolver=resolver, + ) + + with pytest.raises(RuntimeError, match="Unable to resolve AWS credentials"): + provider.get_credentials() + + resolver.credentials = _FakeCredentials("AKIA-SYNTHETIC") + user_name, token = provider.get_credentials() + + assert user_name == "iam-user" + assert token + assert resolver.calls == 2 From 604b9edc533ec236eaa3563e61e5fade4bfd6279 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 26 Aug 2026 15:58:02 -0400 Subject: [PATCH 030/126] test(redis): cover AWS IAM provider install on async cluster nodes No existing test combined aws_iam_auth with startup_nodes, the shape ElastiCache/Valkey Serverless deployments actually use. --- tests/test_litellm/test_redis.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index c99a3a074fb..21c23ba7866 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -488,6 +488,20 @@ def test_azure_wins_over_aws_iam(clean_redis_environment): assert redis_kwargs["redis_connect_func"] is mock_azure.return_value +def test_async_cluster_installs_aws_iam_provider(clean_redis_environment): + startup_nodes = [{"host": "cluster-node", "port": 6379}] + + client = get_redis_async_client( + startup_nodes=startup_nodes, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) + + assert isinstance(client.connection_kwargs["credential_provider"], ElastiCacheIAMCredentialProvider) + + def test_provider_keeps_the_rest_of_the_url_intact(clean_redis_environment): provider = _StubCredentialProvider() From f66891d80f4c7ce4d47a5ddb4ce499e5df21065a Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 26 Aug 2026 19:19:01 -0400 Subject: [PATCH 031/126] fix(redis): type ElastiCache IAM configuration Generated with AI Co-Authored-By: Claude Code --- litellm/_redis.py | 41 ++++++++++--------- litellm/_redis_credential_provider.py | 30 ++++++-------- .../test_redis_credential_provider.py | 25 +++++------ ui/litellm-dashboard/src/lib/http/schema.d.ts | 20 +++++++++ 4 files changed, 65 insertions(+), 51 deletions(-) diff --git a/litellm/_redis.py b/litellm/_redis.py index e289cc3b6c3..2fd8a7100e3 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -279,25 +279,24 @@ def _is_true(value: object | None) -> bool: return value is True or (isinstance(value, str) and value.lower() == "true") -def _build_elasticache_iam_provider(redis_kwargs: dict) -> ElastiCacheIAMCredentialProvider | None: - if not _is_true(redis_kwargs.get("aws_iam_auth")): - return None - - required_settings: Final = { - "aws_iam_user_name": redis_kwargs.get("aws_iam_user_name"), - "aws_iam_cache_name": redis_kwargs.get("aws_iam_cache_name"), - "aws_iam_region": redis_kwargs.get("aws_iam_region") - or get_secret_str("AWS_REGION") - or get_secret_str("AWS_DEFAULT_REGION"), - } - missing_settings: Final = tuple(name for name, value in required_settings.items() if not value) +def _build_elasticache_iam_provider( + user_name: object | None, + cache_name: object | None, + region: object | None, +) -> ElastiCacheIAMCredentialProvider: + required_settings: Final = ( + ("aws_iam_user_name", user_name), + ("aws_iam_cache_name", cache_name), + ("aws_iam_region", region), + ) + missing_settings: Final = tuple(name for name, value in required_settings if not value) if missing_settings: raise ValueError("AWS ElastiCache IAM Redis authentication requires: " + ", ".join(missing_settings)) return ElastiCacheIAMCredentialProvider( - user_name=str(required_settings["aws_iam_user_name"]), - cache_name=str(required_settings["aws_iam_cache_name"]), - region=str(required_settings["aws_iam_region"]), + user_name=str(user_name), + cache_name=str(cache_name), + region=str(region), ) @@ -605,10 +604,14 @@ def _get_redis_client_logic(**env_overrides): "for Redis. Using Azure AD. Remove one to avoid misconfiguration." ) elif _aws_iam_enabled: - aws_provider: Final = _build_elasticache_iam_provider(redis_kwargs) - if aws_provider is not None: - verbose_logger.debug("Setting up AWS ElastiCache IAM authentication for Redis.") - redis_kwargs["credential_provider"] = aws_provider + verbose_logger.debug("Setting up AWS ElastiCache IAM authentication for Redis.") + redis_kwargs["credential_provider"] = _build_elasticache_iam_provider( + user_name=redis_kwargs.get("aws_iam_user_name"), + cache_name=redis_kwargs.get("aws_iam_cache_name"), + region=redis_kwargs.get("aws_iam_region") + or get_secret_str("AWS_REGION") + or get_secret_str("AWS_DEFAULT_REGION"), + ) redis_kwargs.pop("gcp_service_account", None) redis_kwargs.pop("gcp_ssl_ca_certs", None) diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index a20bbfc530c..85759af2a88 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -1,11 +1,19 @@ +from __future__ import annotations + import asyncio import threading import time -from typing import Any, Final, Protocol +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Final, Protocol from urllib.parse import quote from redis.credentials import CredentialProvider +if TYPE_CHECKING: + from botocore.credentials import Credentials +else: + Credentials = Any # rebind-ok: runtime alias for the type-checking-only botocore import + # Azure AD scope for Redis Cache for Azure. AZURE_REDIS_SCOPE: Final = "https://redis.azure.com/.default" @@ -122,38 +130,24 @@ _ELASTICACHE_SERVICE_NAME: Final = "elasticache" _ELASTICACHE_TOKEN_TTL_SECONDS: Final = 900 -class _FrozenBotocoreCredentials(Protocol): - access_key: str - secret_key: str - token: str | None - - -class _BotocoreCredentials(Protocol): - def get_frozen_credentials(self) -> _FrozenBotocoreCredentials: ... - - -class _BotocoreCredentialsResolver(Protocol): - def __call__(self) -> _BotocoreCredentials | None: ... - - class ElastiCacheIAMCredentialProvider(CredentialProvider): def __init__( self, user_name: str, cache_name: str, region: str, - credentials_resolver: _BotocoreCredentialsResolver | None = None, + credentials_resolver: Callable[[], Credentials | None] | None = None, token_lifetime_seconds: int = _ELASTICACHE_TOKEN_TTL_SECONDS, ) -> None: self._user_name = user_name self._cache_name = cache_name self._region = region self._credentials_resolver = credentials_resolver or self._resolve_credentials - self._credentials: _BotocoreCredentials | None = None + self._credentials: Credentials | None = None self._token_lifetime_seconds = token_lifetime_seconds @staticmethod - def _resolve_credentials() -> Any: + def _resolve_credentials() -> Credentials | None: try: import botocore.session except ImportError as e: diff --git a/tests/test_litellm/test_redis_credential_provider.py b/tests/test_litellm/test_redis_credential_provider.py index 6299d4d42e0..de4afb7c313 100644 --- a/tests/test_litellm/test_redis_credential_provider.py +++ b/tests/test_litellm/test_redis_credential_provider.py @@ -4,10 +4,7 @@ from urllib.parse import parse_qs, urlsplit import pytest -from litellm._redis_credential_provider import ( - ElastiCacheIAMCredentialProvider, - _BotocoreCredentials, -) +from litellm._redis_credential_provider import ElastiCacheIAMCredentialProvider class _FakeCredentials: @@ -20,16 +17,6 @@ class _FakeCredentials: return self -class _FakeResolver: - def __init__(self, credentials: _BotocoreCredentials | None) -> None: - self.credentials = credentials - self.calls = 0 - - def __call__(self): - self.calls += 1 - return self.credentials - - class _RotatingFakeCredentials: def __init__(self) -> None: self.calls = 0 @@ -46,6 +33,16 @@ class _RotatingFakeCredentials: ) +class _FakeResolver: + def __init__(self, credentials: _FakeCredentials | _RotatingFakeCredentials | None) -> None: + self.credentials = credentials + self.calls = 0 + + def __call__(self): + self.calls += 1 + return self.credentials + + def test_elasticache_provider_signs_expected_query(): resolver = _FakeResolver(_FakeCredentials("AKIA-SYNTHETIC")) provider = ElastiCacheIAMCredentialProvider( diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6d62ce2b675..9ca0df396b7 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -26357,6 +26357,26 @@ export interface components { * independently of the response-cache backend in `litellm_settings.cache_params`. */ CoordinationRedisParams: { + /** + * Aws Iam Auth + * @description enable AWS ElastiCache IAM authentication + */ + aws_iam_auth?: boolean | string | null; + /** + * Aws Iam Cache Name + * @description AWS ElastiCache cache name + */ + aws_iam_cache_name?: string | null; + /** + * Aws Iam Region + * @description AWS region for ElastiCache IAM authentication + */ + aws_iam_region?: string | null; + /** + * Aws Iam User Name + * @description AWS ElastiCache IAM user name + */ + aws_iam_user_name?: string | null; /** * Host * @description Redis hostname From bb043fe29a7f680f8a5aee1ba5998f3fdb26e561 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 26 Aug 2026 19:48:03 -0400 Subject: [PATCH 032/126] test(redis): cover ElastiCache IAM failures Generated with AI Co-Authored-By: Claude Code --- tests/test_litellm/test_redis.py | 52 ++++++-------- .../test_redis_credential_provider.py | 70 +++++++++++++++++++ 2 files changed, 93 insertions(+), 29 deletions(-) diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index 21c23ba7866..43fb7563233 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -455,37 +455,33 @@ def test_explicit_provider_wins_over_aws_iam(clean_redis_environment): def test_gcp_wins_over_aws_iam(clean_redis_environment): - with patch("litellm._redis.create_gcp_iam_redis_connect_func") as mock_gcp: - mock_gcp.return_value = _gcp_marker_callback() - redis_kwargs = _get_redis_client_logic( - host="cache.example.com", - port=6379, - gcp_service_account="sa@example.com", - aws_iam_auth=True, - aws_iam_user_name="iam-user", - aws_iam_cache_name="cache.example.com", - aws_iam_region="us-east-1", - ) + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + gcp_service_account="sa@example.com", + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) assert "credential_provider" not in redis_kwargs - assert redis_kwargs["redis_connect_func"] is mock_gcp.return_value + assert redis_kwargs["redis_connect_func"]._gcp_service_account == "sa@example.com" def test_azure_wins_over_aws_iam(clean_redis_environment): - with patch("litellm._redis.create_azure_ad_redis_connect_func") as mock_azure: - mock_azure.return_value = MagicMock() - redis_kwargs = _get_redis_client_logic( - host="cache.example.com", - port=6379, - azure_redis_ad_token="true", - aws_iam_auth=True, - aws_iam_user_name="iam-user", - aws_iam_cache_name="cache.example.com", - aws_iam_region="us-east-1", - ) + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + azure_redis_ad_token="true", + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) assert "credential_provider" not in redis_kwargs - assert redis_kwargs["redis_connect_func"] is mock_azure.return_value + assert redis_kwargs["redis_connect_func"]._azure_redis_ad_token is True def test_async_cluster_installs_aws_iam_provider(clean_redis_environment): @@ -1732,11 +1728,9 @@ def test_async_sentinel_keeps_the_credential_provider_off_the_monitors(markers, "redis_connect_func": SimpleNamespace(**markers), } - with ( - patch("litellm._redis.async_redis.Sentinel") as mock_sentinel_cls, - patch("litellm._redis._get_redis_client_logic", return_value=redis_kwargs), - ): - get_redis_async_client() + with patch("litellm._redis.async_redis.Sentinel") as mock_sentinel_cls: + with patch("litellm._redis._get_redis_client_logic", return_value=redis_kwargs): + get_redis_async_client() sentinel_kwargs = mock_sentinel_cls.call_args[1]["sentinel_kwargs"] assert sentinel_kwargs["password"] == sentinel_password diff --git a/tests/test_litellm/test_redis_credential_provider.py b/tests/test_litellm/test_redis_credential_provider.py index de4afb7c313..336e30e12c6 100644 --- a/tests/test_litellm/test_redis_credential_provider.py +++ b/tests/test_litellm/test_redis_credential_provider.py @@ -1,4 +1,6 @@ import asyncio +import builtins +import sys from types import SimpleNamespace from urllib.parse import parse_qs, urlsplit @@ -87,6 +89,41 @@ def test_elasticache_provider_resolves_credentials_once_but_refreshes_signature( assert rotating_credentials.calls == 3 +def test_elasticache_provider_uses_botocore_session_credentials(monkeypatch): + credentials = _FakeCredentials("AKIA-SYNTHETIC") + monkeypatch.setattr("botocore.session.get_session", lambda: SimpleNamespace(get_credentials=lambda: credentials)) + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + ) + + user_name, token = provider.get_credentials() + + assert user_name == "iam-user" + assert "AKIA-SYNTHETIC" in token + + +def test_elasticache_provider_reports_missing_botocore(monkeypatch): + original_import = builtins.__import__ + + def import_without_botocore(name, *args, **kwargs): + if name == "botocore.session": + raise ImportError("synthetic missing dependency") + return original_import(name, *args, **kwargs) + + monkeypatch.delitem(sys.modules, "botocore.session", raising=False) + monkeypatch.setattr(builtins, "__import__", import_without_botocore) + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + ) + + with pytest.raises(ImportError, match="pip install boto3"): + provider.get_credentials() + + def test_elasticache_provider_reports_missing_credentials(): provider = ElastiCacheIAMCredentialProvider( user_name="iam-user", @@ -99,6 +136,39 @@ def test_elasticache_provider_reports_missing_credentials(): provider.get_credentials() +def test_elasticache_provider_reports_missing_frozen_credentials(): + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + credentials_resolver=_FakeResolver(SimpleNamespace(get_frozen_credentials=lambda: None)), + ) + + with pytest.raises(RuntimeError, match="Unable to resolve AWS credentials"): + provider.get_credentials() + + +def test_elasticache_provider_reports_missing_signing_dependency(monkeypatch): + original_import = builtins.__import__ + + def import_without_botocore_auth(name, *args, **kwargs): + if name == "botocore.auth": + raise ImportError("synthetic missing dependency") + return original_import(name, *args, **kwargs) + + monkeypatch.delitem(sys.modules, "botocore.auth", raising=False) + monkeypatch.setattr(builtins, "__import__", import_without_botocore_auth) + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + credentials_resolver=_FakeResolver(_FakeCredentials("AKIA-SYNTHETIC")), + ) + + with pytest.raises(ImportError, match="pip install boto3"): + provider.get_credentials() + + def test_elasticache_provider_recovers_after_a_failed_resolution(): resolver = _FakeResolver(None) provider = ElastiCacheIAMCredentialProvider( From 921c263876600776be8dd69980f683b2c9ea3940 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 26 Aug 2026 20:21:03 -0400 Subject: [PATCH 033/126] fix(proxy): validate coordination Redis mappings Generated with AI Co-Authored-By: Claude Code --- .../management_endpoints/coordination_redis_endpoints.py | 2 +- litellm/proxy/proxy_server.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/management_endpoints/coordination_redis_endpoints.py b/litellm/proxy/management_endpoints/coordination_redis_endpoints.py index 86ce336c7a3..88dc09ab001 100644 --- a/litellm/proxy/management_endpoints/coordination_redis_endpoints.py +++ b/litellm/proxy/management_endpoints/coordination_redis_endpoints.py @@ -140,7 +140,7 @@ def _merge_over_saved( def _validated_params(settings: Mapping[str, object]) -> CoordinationRedisParams: """Validate settings the way startup does: resolve env refs, then require a connection target.""" try: - params: Final = CoordinationRedisParams(**_resolve_env_refs(settings)) + params: Final = CoordinationRedisParams.model_validate(_resolve_env_refs(settings)) except ValidationError as e: invalid_fields: Final = sorted({str(error["loc"][0]) for error in e.errors() if error["loc"]}) raise HTTPException( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9269fd48e6c..c21df8ce931 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4906,7 +4906,7 @@ class ProxyConfig: if not isinstance(raw_params, dict): raise ValueError("general_settings.coordination_redis must be a mapping of Redis connection params") - coordination_params: Final = CoordinationRedisParams(**_resolve_coordination_redis_env_refs(raw_params)) + coordination_params: Final = CoordinationRedisParams.model_validate(_resolve_coordination_redis_env_refs(raw_params)) if not coordination_params.has_connection_target(): raise ValueError( "general_settings.coordination_redis needs a connection target: " @@ -9234,7 +9234,7 @@ class ProxyStartupEvent: if persisted is None: return None - coordination_params: Final = CoordinationRedisParams(**_resolve_coordination_redis_env_refs(persisted)) + coordination_params: Final = CoordinationRedisParams.model_validate(_resolve_coordination_redis_env_refs(persisted)) if not coordination_params.has_connection_target(): verbose_proxy_logger.warning( "coordination_redis saved in the database names no connection target; ignoring it." From 9763bd80ae442533d234e733cf334e3e1565681e Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 26 Aug 2026 20:26:26 -0400 Subject: [PATCH 034/126] style(proxy): format coordination Redis validation Generated with AI Co-Authored-By: Claude Code --- litellm/proxy/proxy_server.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c21df8ce931..de10fde45c0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4906,7 +4906,9 @@ class ProxyConfig: if not isinstance(raw_params, dict): raise ValueError("general_settings.coordination_redis must be a mapping of Redis connection params") - coordination_params: Final = CoordinationRedisParams.model_validate(_resolve_coordination_redis_env_refs(raw_params)) + coordination_params: Final = CoordinationRedisParams.model_validate( + _resolve_coordination_redis_env_refs(raw_params) + ) if not coordination_params.has_connection_target(): raise ValueError( "general_settings.coordination_redis needs a connection target: " @@ -9234,7 +9236,9 @@ class ProxyStartupEvent: if persisted is None: return None - coordination_params: Final = CoordinationRedisParams.model_validate(_resolve_coordination_redis_env_refs(persisted)) + coordination_params: Final = CoordinationRedisParams.model_validate( + _resolve_coordination_redis_env_refs(persisted) + ) if not coordination_params.has_connection_target(): verbose_proxy_logger.warning( "coordination_redis saved in the database names no connection target; ignoring it." From a7ad6023d6a52ed035b638c4b3ab867c27a3b65b Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 26 Aug 2026 21:14:07 -0400 Subject: [PATCH 035/126] ci: retry CodSpeed result upload Generated with AI Co-Authored-By: Claude Code From 0217a137fe3f8793163140fdb4d2f1fa4d132bbf Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Thu, 27 Aug 2026 13:49:12 -0400 Subject: [PATCH 036/126] fix(redis): require TLS for ElastiCache IAM auth --- litellm/_redis.py | 9 ++++ tests/test_litellm/test_redis.py | 78 ++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/litellm/_redis.py b/litellm/_redis.py index 2fd8a7100e3..50900b2977b 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -279,6 +279,13 @@ def _is_true(value: object | None) -> bool: return value is True or (isinstance(value, str) and value.lower() == "true") +def _uses_tls(redis_kwargs: Mapping[str, object]) -> bool: + if redis_kwargs.get("startup_nodes") is not None: + return _is_true(redis_kwargs.get("ssl")) + url: Final = redis_kwargs.get("url") + return urlsplit(url).scheme.lower() == "rediss" if isinstance(url, str) else _is_true(redis_kwargs.get("ssl")) + + def _build_elasticache_iam_provider( user_name: object | None, cache_name: object | None, @@ -604,6 +611,8 @@ def _get_redis_client_logic(**env_overrides): "for Redis. Using Azure AD. Remove one to avoid misconfiguration." ) elif _aws_iam_enabled: + if not _uses_tls(redis_kwargs): + raise ValueError("AWS ElastiCache IAM Redis authentication requires TLS") verbose_logger.debug("Setting up AWS ElastiCache IAM authentication for Redis.") redis_kwargs["credential_provider"] = _build_elasticache_iam_provider( user_name=redis_kwargs.get("aws_iam_user_name"), diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index 43fb7563233..f2eda9a5753 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -319,6 +319,7 @@ def test_aws_iam_environment_settings_install_provider(clean_redis_environment, monkeypatch.setenv("REDIS_AWS_IAM_USER_NAME", "iam-user") monkeypatch.setenv("REDIS_AWS_IAM_CACHE_NAME", "cache.example.com") monkeypatch.setenv("REDIS_AWS_IAM_REGION", "us-east-1") + monkeypatch.setenv("REDIS_SSL", "true") redis_kwargs = _get_redis_client_logic(host="cache.example.com", port=6379) @@ -326,6 +327,77 @@ def test_aws_iam_environment_settings_install_provider(clean_redis_environment, assert not {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} & redis_kwargs.keys() +@pytest.mark.parametrize( + "transport", + [ + pytest.param({"host": "cache.example.com", "port": 6379}, id="host_without_ssl"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": False}, id="host_ssl_false"), + pytest.param({"url": "redis://cache.example.com:6379", "ssl": True}, id="plaintext_url"), + pytest.param( + {"startup_nodes": [{"host": "cache.example.com", "port": 6379}]}, + id="cluster_without_ssl", + ), + pytest.param( + { + "url": "rediss://cache.example.com:6379", + "startup_nodes": [{"host": "cache.example.com", "port": 6379}], + }, + id="cluster_without_ssl_ignores_url_scheme", + ), + pytest.param( + { + "sentinel_nodes": [("sentinel.example.com", 26379)], + "service_name": "cache", + }, + id="sentinel_without_ssl", + ), + ], +) +def test_aws_iam_auth_rejects_non_tls_connections(clean_redis_environment, transport): + with pytest.raises(ValueError, match="requires TLS"): + _get_redis_client_logic( + **transport, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) + + +@pytest.mark.parametrize( + "transport", + [ + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": True}, id="host"), + pytest.param({"url": "rediss://cache.example.com:6379"}, id="url"), + pytest.param( + { + "startup_nodes": [{"host": "cache.example.com", "port": 6379}], + "ssl": True, + }, + id="cluster", + ), + pytest.param( + { + "sentinel_nodes": [("sentinel.example.com", 26379)], + "service_name": "cache", + "ssl": True, + }, + id="sentinel", + ), + ], +) +def test_aws_iam_auth_accepts_tls_connections(clean_redis_environment, transport): + redis_kwargs = _get_redis_client_logic( + **transport, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) + + assert isinstance(redis_kwargs["credential_provider"], ElastiCacheIAMCredentialProvider) + + def test_aws_iam_settings_are_removed_for_url_and_static_credentials(clean_redis_environment): redis_kwargs = _get_redis_client_logic( url="rediss://url-user:url-pass@cache.example.com:6380", @@ -351,6 +423,7 @@ def test_aws_iam_missing_setting_fails_closed(clean_redis_environment, missing): "aws_iam_user_name": "iam-user", "aws_iam_cache_name": "cache.example.com", "aws_iam_region": "us-east-1", + "ssl": True, } settings[missing] = None @@ -365,6 +438,7 @@ def test_aws_iam_region_falls_back_to_environment(clean_redis_environment, monke redis_kwargs = _get_redis_client_logic( host="cache.example.com", port=6379, + ssl=True, aws_iam_auth=True, aws_iam_user_name="iam-user", aws_iam_cache_name="cache.example.com", @@ -382,6 +456,7 @@ def test_aws_iam_region_prefers_aws_region_over_default_region(clean_redis_envir redis_kwargs = _get_redis_client_logic( host="cache.example.com", port=6379, + ssl=True, aws_iam_auth=True, aws_iam_user_name="iam-user", aws_iam_cache_name="cache.example.com", @@ -396,6 +471,7 @@ def test_aws_iam_region_prefers_explicit_over_environment(clean_redis_environmen redis_kwargs = _get_redis_client_logic( host="cache.example.com", port=6379, + ssl=True, aws_iam_auth=True, aws_iam_user_name="iam-user", aws_iam_cache_name="cache.example.com", @@ -409,6 +485,7 @@ def test_aws_iam_settings_map_to_distinct_provider_fields(clean_redis_environmen redis_kwargs = _get_redis_client_logic( host="cache.example.com", port=6379, + ssl=True, aws_iam_auth=True, aws_iam_user_name="iam-user-value", aws_iam_cache_name="iam-cache-value", @@ -489,6 +566,7 @@ def test_async_cluster_installs_aws_iam_provider(clean_redis_environment): client = get_redis_async_client( startup_nodes=startup_nodes, + ssl=True, aws_iam_auth=True, aws_iam_user_name="iam-user", aws_iam_cache_name="cache.example.com", From f1cd2b03caac2c544866be984089039aa1228956 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Mon, 31 Aug 2026 13:49:55 -0400 Subject: [PATCH 037/126] fix(redis): type signed ElastiCache IAM URL --- litellm/_redis_credential_provider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index 85759af2a88..4d728ef21b6 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -4,7 +4,7 @@ import asyncio import threading import time from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Final, Protocol +from typing import TYPE_CHECKING, Any, Final, Protocol, cast from urllib.parse import quote from redis.credentials import CredentialProvider @@ -185,7 +185,7 @@ class ElastiCacheIAMCredentialProvider(CredentialProvider): self._region, expires=self._token_lifetime_seconds, ).add_auth(request) - return self._user_name, request.url.removeprefix("https://") + return self._user_name, cast(str, request.url).removeprefix("https://") def get_credentials(self) -> tuple[str, str]: return self._get_credentials() From aca81c263ccd6660bffcf05115e4971eb92412c8 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Mon, 31 Aug 2026 14:07:04 -0400 Subject: [PATCH 038/126] fix(redis): validate signed ElastiCache IAM URL --- litellm/_redis_credential_provider.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index 4d728ef21b6..4441d318373 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -4,7 +4,7 @@ import asyncio import threading import time from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Final, Protocol, cast +from typing import TYPE_CHECKING, Any, Final, Protocol from urllib.parse import quote from redis.credentials import CredentialProvider @@ -185,7 +185,10 @@ class ElastiCacheIAMCredentialProvider(CredentialProvider): self._region, expires=self._token_lifetime_seconds, ).add_auth(request) - return self._user_name, cast(str, request.url).removeprefix("https://") + signed_url: Final = request.url + if signed_url is None: + raise RuntimeError("Unable to generate AWS ElastiCache IAM credentials") + return self._user_name, signed_url.removeprefix("https://") def get_credentials(self) -> tuple[str, str]: return self._get_credentials() From cd4073895ff9884d6fa77337fe6ac4c62dc029ec Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 9 Sep 2026 10:03:26 -0400 Subject: [PATCH 039/126] refactor(redis): type botocore credentials without Any Deferred annotation evaluation keeps the type-checking-only botocore import off the runtime path, so the alias only reintroduced typing.Any, which the strict ruff budget now bans --- litellm/_redis_credential_provider.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index 4441d318373..15f625dc8b4 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -4,15 +4,13 @@ import asyncio import threading import time from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Final, Protocol +from typing import TYPE_CHECKING, Final, Protocol from urllib.parse import quote from redis.credentials import CredentialProvider if TYPE_CHECKING: from botocore.credentials import Credentials -else: - Credentials = Any # rebind-ok: runtime alias for the type-checking-only botocore import # Azure AD scope for Redis Cache for Azure. AZURE_REDIS_SCOPE: Final = "https://redis.azure.com/.default" From 2b0711e4f518dc796395972896aa5c3e6608b965 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 9 Sep 2026 10:06:22 -0400 Subject: [PATCH 040/126] style(redis): restore the Azure AD marker comment The comment documents that the raw Azure client id, tenant id and secret are deliberately kept off the connect function, so this branch should never have dropped it --- litellm/_redis.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/litellm/_redis.py b/litellm/_redis.py index 50900b2977b..d77043a6dfd 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -598,6 +598,11 @@ def _get_redis_client_logic(**env_overrides): azure_tenant_id=_azure_tenant_id, azure_client_secret=_azure_client_secret, ) + # Marker for async paths to detect Azure AD auth. The live credential + # object is attached separately as `_azure_credential` by + # `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret + # are intentionally NOT exposed on the function to avoid leaking + # credentials via inspection or logging. redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True if _aws_iam_enabled and _gcp_service_account is not None: From bf73c49d737b0ee1d4274a36a1846a8628de3478 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 9 Sep 2026 10:17:53 -0400 Subject: [PATCH 041/126] refactor(redis): drop unreachable frozen credentials guard --- litellm/_redis_credential_provider.py | 2 -- tests/test_litellm/test_redis_credential_provider.py | 12 ------------ 2 files changed, 14 deletions(-) diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index 15f625dc8b4..dd49a6793ec 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -162,8 +162,6 @@ class ElastiCacheIAMCredentialProvider(CredentialProvider): self._credentials = credentials frozen_credentials: Final = credentials.get_frozen_credentials() - if frozen_credentials is None: - raise RuntimeError("Unable to resolve AWS credentials for ElastiCache IAM Redis authentication") try: from botocore.auth import SigV4QueryAuth diff --git a/tests/test_litellm/test_redis_credential_provider.py b/tests/test_litellm/test_redis_credential_provider.py index 336e30e12c6..9a58722b04e 100644 --- a/tests/test_litellm/test_redis_credential_provider.py +++ b/tests/test_litellm/test_redis_credential_provider.py @@ -136,18 +136,6 @@ def test_elasticache_provider_reports_missing_credentials(): provider.get_credentials() -def test_elasticache_provider_reports_missing_frozen_credentials(): - provider = ElastiCacheIAMCredentialProvider( - user_name="iam-user", - cache_name="cache.example.com", - region="us-east-1", - credentials_resolver=_FakeResolver(SimpleNamespace(get_frozen_credentials=lambda: None)), - ) - - with pytest.raises(RuntimeError, match="Unable to resolve AWS credentials"): - provider.get_credentials() - - def test_elasticache_provider_reports_missing_signing_dependency(monkeypatch): original_import = builtins.__import__ From c4fc20bcf933606d0ffc91a47036dc16df7059ea Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 9 Sep 2026 14:43:58 -0400 Subject: [PATCH 042/126] fix(redis): accept every truthy flag and sign serverless ElastiCache caches The ElastiCache IAM gate read `aws_iam_auth` and `ssl` with a helper that only accepted the literal string "true", while the kwarg coercion that runs later accepts "true", "1" and "yes". Type coercion happens after the gate, so `REDIS_AWS_IAM_AUTH=1` silently skipped IAM auth and `REDIS_SSL=1` made the "requires TLS" check fail closed on a connection that was in fact TLS. Both helpers now share `_str_to_bool`. AWS signs serverless cache tokens with an extra `ResourceType=ServerlessCache` query parameter, so tokens minted for a serverless cache were rejected. Adds an `aws_iam_serverless` setting (`REDIS_AWS_IAM_SERVERLESS`) that puts the parameter into the signed URL, and lowercases the cache name because ElastiCache lowercases it at creation time. --- litellm/_redis.py | 51 ++++++------ litellm/_redis_credential_provider.py | 17 ++-- litellm/proxy/_types.py | 3 + .../cache_settings_endpoints.py | 9 ++ .../coordination_redis_endpoints.py | 8 ++ tests/test_litellm/test_redis.py | 82 +++++++++++++++++-- .../test_redis_credential_provider.py | 54 ++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 ++ 8 files changed, 192 insertions(+), 37 deletions(-) diff --git a/litellm/_redis.py b/litellm/_redis.py index d77043a6dfd..c5acdcb038b 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -39,6 +39,14 @@ from ._logging import verbose_logger AZURE_REDIS_SCOPE: Final = "https://redis.azure.com/.default" +_AWS_IAM_KWARG_NAMES: Final = ( + "aws_iam_auth", + "aws_iam_user_name", + "aws_iam_cache_name", + "aws_iam_region", + "aws_iam_serverless", +) + def _unwrapped_init_args(cls: type) -> frozenset[str]: """Every parameter on a single class's own ``__init__``, decorator-unwrapped. @@ -76,10 +84,7 @@ def _get_redis_kwargs(): "azure_client_id", "azure_tenant_id", "azure_client_secret", - "aws_iam_auth", - "aws_iam_user_name", - "aws_iam_cache_name", - "aws_iam_region", + *_AWS_IAM_KWARG_NAMES, } available_args: Final = {x for x in _unwrapped_init_args(redis.Redis) if x not in exclude_args} | include_args @@ -275,22 +280,25 @@ def _redis_kwargs_from_environment(): return return_dict -def _is_true(value: object | None) -> bool: - return value is True or (isinstance(value, str) and value.lower() == "true") +def _coerces_to_true(value: object | None) -> bool: + return _str_to_bool(value) if isinstance(value, str) else bool(value) def _uses_tls(redis_kwargs: Mapping[str, object]) -> bool: if redis_kwargs.get("startup_nodes") is not None: - return _is_true(redis_kwargs.get("ssl")) + return _coerces_to_true(redis_kwargs.get("ssl")) url: Final = redis_kwargs.get("url") - return urlsplit(url).scheme.lower() == "rediss" if isinstance(url, str) else _is_true(redis_kwargs.get("ssl")) + if isinstance(url, str): + return urlsplit(url).scheme.lower() == "rediss" + return _coerces_to_true(redis_kwargs.get("ssl")) -def _build_elasticache_iam_provider( - user_name: object | None, - cache_name: object | None, - region: object | None, -) -> ElastiCacheIAMCredentialProvider: +def _build_elasticache_iam_provider(redis_kwargs: Mapping[str, object]) -> ElastiCacheIAMCredentialProvider: + user_name: Final = redis_kwargs.get("aws_iam_user_name") + cache_name: Final = redis_kwargs.get("aws_iam_cache_name") + region: Final = ( + redis_kwargs.get("aws_iam_region") or get_secret_str("AWS_REGION") or get_secret_str("AWS_DEFAULT_REGION") + ) required_settings: Final = ( ("aws_iam_user_name", user_name), ("aws_iam_cache_name", cache_name), @@ -304,6 +312,7 @@ def _build_elasticache_iam_provider( user_name=str(user_name), cache_name=str(cache_name), region=str(region), + is_serverless=_coerces_to_true(redis_kwargs.get("aws_iam_serverless")), ) @@ -577,7 +586,7 @@ def _get_redis_client_logic(**env_overrides): _azure_redis_ad_token: Final = redis_kwargs.get("azure_redis_ad_token") or get_secret("REDIS_AZURE_AD_TOKEN") _azure_ad_enabled: Final = _azure_redis_ad_token is not None and str(_azure_redis_ad_token).lower() == "true" - _aws_iam_enabled: Final = _is_true(redis_kwargs.get("aws_iam_auth")) + _aws_iam_enabled: Final = _coerces_to_true(redis_kwargs.get("aws_iam_auth")) if _azure_ad_enabled and _gcp_service_account is not None: verbose_logger.warning( @@ -619,13 +628,7 @@ def _get_redis_client_logic(**env_overrides): if not _uses_tls(redis_kwargs): raise ValueError("AWS ElastiCache IAM Redis authentication requires TLS") verbose_logger.debug("Setting up AWS ElastiCache IAM authentication for Redis.") - redis_kwargs["credential_provider"] = _build_elasticache_iam_provider( - user_name=redis_kwargs.get("aws_iam_user_name"), - cache_name=redis_kwargs.get("aws_iam_cache_name"), - region=redis_kwargs.get("aws_iam_region") - or get_secret_str("AWS_REGION") - or get_secret_str("AWS_DEFAULT_REGION"), - ) + redis_kwargs["credential_provider"] = _build_elasticache_iam_provider(redis_kwargs) redis_kwargs.pop("gcp_service_account", None) redis_kwargs.pop("gcp_ssl_ca_certs", None) @@ -635,10 +638,8 @@ def _get_redis_client_logic(**env_overrides): redis_kwargs.pop("azure_client_id", None) redis_kwargs.pop("azure_tenant_id", None) redis_kwargs.pop("azure_client_secret", None) - redis_kwargs.pop("aws_iam_auth", None) - redis_kwargs.pop("aws_iam_user_name", None) - redis_kwargs.pop("aws_iam_cache_name", None) - redis_kwargs.pop("aws_iam_region", None) + for aws_iam_key in _AWS_IAM_KWARG_NAMES: + redis_kwargs.pop(aws_iam_key, None) if redis_kwargs.get("credential_provider") is not None: redis_kwargs.pop("redis_connect_func", None) diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index dd49a6793ec..7d90f944657 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -5,7 +5,7 @@ import threading import time from collections.abc import Callable from typing import TYPE_CHECKING, Final, Protocol -from urllib.parse import quote +from urllib.parse import urlencode from redis.credentials import CredentialProvider @@ -126,6 +126,7 @@ class GCPIAMCredentialProvider(CredentialProvider): _ELASTICACHE_SERVICE_NAME: Final = "elasticache" _ELASTICACHE_TOKEN_TTL_SECONDS: Final = 900 +_ELASTICACHE_SERVERLESS_RESOURCE_TYPE: Final = "ServerlessCache" class ElastiCacheIAMCredentialProvider(CredentialProvider): @@ -134,12 +135,14 @@ class ElastiCacheIAMCredentialProvider(CredentialProvider): user_name: str, cache_name: str, region: str, + is_serverless: bool = False, credentials_resolver: Callable[[], Credentials | None] | None = None, token_lifetime_seconds: int = _ELASTICACHE_TOKEN_TTL_SECONDS, ) -> None: self._user_name = user_name - self._cache_name = cache_name + self._cache_name = cache_name.lower() self._region = region + self._is_serverless = is_serverless self._credentials_resolver = credentials_resolver or self._resolve_credentials self._credentials: Credentials | None = None self._token_lifetime_seconds = token_lifetime_seconds @@ -171,10 +174,14 @@ class ElastiCacheIAMCredentialProvider(CredentialProvider): "botocore is required for ElastiCache IAM Redis authentication. Install it with: pip install boto3" ) from e - request: Final = AWSRequest( - method="GET", - url=(f"https://{self._cache_name}/?Action=connect&User={quote(self._user_name, safe='')}"), + query: Final = urlencode( + ( + ("Action", "connect"), + ("User", self._user_name), + *((("ResourceType", _ELASTICACHE_SERVERLESS_RESOURCE_TYPE),) if self._is_serverless else ()), + ) ) + request: Final = AWSRequest(method="GET", url=f"https://{self._cache_name}/?{query}") SigV4QueryAuth( frozen_credentials, _ELASTICACHE_SERVICE_NAME, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 99103ff3706..07e4515059a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2431,6 +2431,9 @@ class CoordinationRedisParams(LiteLLMPydanticObjectBase): aws_iam_user_name: str | None = Field(None, description="AWS ElastiCache IAM user name") aws_iam_cache_name: str | None = Field(None, description="AWS ElastiCache cache name") aws_iam_region: str | None = Field(None, description="AWS region for ElastiCache IAM authentication") + aws_iam_serverless: bool | str | None = Field( + None, description="the ElastiCache cache is serverless rather than a self-designed cluster" + ) def has_connection_target(self) -> bool: return any(value is not None for value in (self.host, self.url, self.startup_nodes, self.sentinel_nodes)) diff --git a/litellm/types/management_endpoints/cache_settings_endpoints.py b/litellm/types/management_endpoints/cache_settings_endpoints.py index 10aece6efa4..cb05f3a50ac 100644 --- a/litellm/types/management_endpoints/cache_settings_endpoints.py +++ b/litellm/types/management_endpoints/cache_settings_endpoints.py @@ -273,4 +273,13 @@ CACHE_SETTINGS_FIELDS: Final[list[CacheSettingsField]] = [ ui_field_name="AWS IAM Region", redis_type=None, ), + CacheSettingsField( + field_name="aws_iam_serverless", + field_type="Boolean", + field_value=None, + field_description="The ElastiCache cache is serverless rather than a self-designed cluster", + field_default=False, + ui_field_name="AWS IAM Serverless Cache", + redis_type=None, + ), ] diff --git a/litellm/types/management_endpoints/coordination_redis_endpoints.py b/litellm/types/management_endpoints/coordination_redis_endpoints.py index 86b32752cbb..d70a921a22f 100644 --- a/litellm/types/management_endpoints/coordination_redis_endpoints.py +++ b/litellm/types/management_endpoints/coordination_redis_endpoints.py @@ -131,4 +131,12 @@ COORDINATION_REDIS_SETTINGS_FIELDS: Final[list[CoordinationRedisSettingsField]] ui_field_name="AWS IAM Region", section="connection", ), + CoordinationRedisSettingsField( + field_name="aws_iam_serverless", + field_type="Boolean", + field_description="The ElastiCache cache is serverless rather than a self-designed cluster", + field_default=False, + ui_field_name="AWS IAM Serverless Cache", + section="connection", + ), ] diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index f2eda9a5753..5337ed825a7 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -113,15 +113,25 @@ def test_credential_provider_is_not_environment_derived(): assert "credential_provider" not in mapping.values() +_AWS_IAM_SETTINGS = { + "aws_iam_auth", + "aws_iam_user_name", + "aws_iam_cache_name", + "aws_iam_region", + "aws_iam_serverless", +} + + def test_aws_iam_settings_are_environment_derived(): allowed = _get_redis_kwargs() mapping = _get_redis_env_kwarg_mapping() - assert {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} <= allowed + assert _AWS_IAM_SETTINGS <= allowed assert mapping["REDIS_AWS_IAM_AUTH"] == "aws_iam_auth" assert mapping["REDIS_AWS_IAM_USER_NAME"] == "aws_iam_user_name" assert mapping["REDIS_AWS_IAM_CACHE_NAME"] == "aws_iam_cache_name" assert mapping["REDIS_AWS_IAM_REGION"] == "aws_iam_region" + assert mapping["REDIS_AWS_IAM_SERVERLESS"] == "aws_iam_serverless" def test_sync_direct_preserves_credential_provider_identity(clean_redis_environment): @@ -319,12 +329,15 @@ def test_aws_iam_environment_settings_install_provider(clean_redis_environment, monkeypatch.setenv("REDIS_AWS_IAM_USER_NAME", "iam-user") monkeypatch.setenv("REDIS_AWS_IAM_CACHE_NAME", "cache.example.com") monkeypatch.setenv("REDIS_AWS_IAM_REGION", "us-east-1") - monkeypatch.setenv("REDIS_SSL", "true") + monkeypatch.setenv("REDIS_AWS_IAM_SERVERLESS", "1") + monkeypatch.setenv("REDIS_SSL", "1") redis_kwargs = _get_redis_client_logic(host="cache.example.com", port=6379) - assert isinstance(redis_kwargs["credential_provider"], ElastiCacheIAMCredentialProvider) - assert not {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} & redis_kwargs.keys() + provider = redis_kwargs["credential_provider"] + assert isinstance(provider, ElastiCacheIAMCredentialProvider) + assert provider._is_serverless is True + assert not _AWS_IAM_SETTINGS & redis_kwargs.keys() @pytest.mark.parametrize( @@ -332,6 +345,9 @@ def test_aws_iam_environment_settings_install_provider(clean_redis_environment, [ pytest.param({"host": "cache.example.com", "port": 6379}, id="host_without_ssl"), pytest.param({"host": "cache.example.com", "port": 6379, "ssl": False}, id="host_ssl_false"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "false"}, id="host_ssl_false_string"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "0"}, id="host_ssl_zero_string"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "no"}, id="host_ssl_no_string"), pytest.param({"url": "redis://cache.example.com:6379", "ssl": True}, id="plaintext_url"), pytest.param( {"startup_nodes": [{"host": "cache.example.com", "port": 6379}]}, @@ -368,6 +384,13 @@ def test_aws_iam_auth_rejects_non_tls_connections(clean_redis_environment, trans "transport", [ pytest.param({"host": "cache.example.com", "port": 6379, "ssl": True}, id="host"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "true"}, id="host_ssl_true_string"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "1"}, id="host_ssl_one_string"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "yes"}, id="host_ssl_yes_string"), + pytest.param( + {"startup_nodes": [{"host": "cache.example.com", "port": 6379}], "ssl": "1"}, + id="cluster_ssl_one_string", + ), pytest.param({"url": "rediss://cache.example.com:6379"}, id="url"), pytest.param( { @@ -413,7 +436,7 @@ def test_aws_iam_settings_are_removed_for_url_and_static_credentials(clean_redis assert redis_kwargs["url"] == "rediss://cache.example.com:6380" assert "username" not in redis_kwargs assert "password" not in redis_kwargs - assert not {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} & redis_kwargs.keys() + assert not _AWS_IAM_SETTINGS & redis_kwargs.keys() @pytest.mark.parametrize("missing", ["aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"]) @@ -499,7 +522,7 @@ def test_aws_iam_settings_map_to_distinct_provider_fields(clean_redis_environmen assert provider._region == "iam-region-value" -@pytest.mark.parametrize("aws_iam_auth", [False, "false"]) +@pytest.mark.parametrize("aws_iam_auth", [None, False, "", "false", "0", "no"]) def test_aws_iam_auth_disabled_does_not_install_provider(clean_redis_environment, aws_iam_auth): redis_kwargs = _get_redis_client_logic( host="cache.example.com", @@ -511,7 +534,52 @@ def test_aws_iam_auth_disabled_does_not_install_provider(clean_redis_environment ) assert "credential_provider" not in redis_kwargs - assert not {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} & redis_kwargs.keys() + assert not _AWS_IAM_SETTINGS & redis_kwargs.keys() + + +@pytest.mark.parametrize("aws_iam_auth", [True, "true", "True", "TRUE", "1", "yes"]) +def test_aws_iam_auth_enabled_by_any_truthy_flag(clean_redis_environment, aws_iam_auth): + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + ssl=True, + aws_iam_auth=aws_iam_auth, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache-name", + aws_iam_region="us-east-1", + ) + + assert isinstance(redis_kwargs["credential_provider"], ElastiCacheIAMCredentialProvider) + + +@pytest.mark.parametrize( + "aws_iam_serverless, expected", + [ + pytest.param(None, False, id="unset"), + pytest.param(False, False, id="bool_false"), + pytest.param("false", False, id="string_false"), + pytest.param("0", False, id="string_zero"), + pytest.param(True, True, id="bool_true"), + pytest.param("true", True, id="string_true"), + pytest.param("1", True, id="string_one"), + ], +) +def test_aws_iam_serverless_flag_reaches_the_provider(clean_redis_environment, aws_iam_serverless, expected): + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + ssl=True, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache-name", + aws_iam_region="us-east-1", + aws_iam_serverless=aws_iam_serverless, + ) + + provider = redis_kwargs["credential_provider"] + assert isinstance(provider, ElastiCacheIAMCredentialProvider) + assert provider._is_serverless is expected + assert "aws_iam_serverless" not in redis_kwargs def test_explicit_provider_wins_over_aws_iam(clean_redis_environment): diff --git a/tests/test_litellm/test_redis_credential_provider.py b/tests/test_litellm/test_redis_credential_provider.py index 9a58722b04e..e71fc5d530d 100644 --- a/tests/test_litellm/test_redis_credential_provider.py +++ b/tests/test_litellm/test_redis_credential_provider.py @@ -175,3 +175,57 @@ def test_elasticache_provider_recovers_after_a_failed_resolution(): assert user_name == "iam-user" assert token assert resolver.calls == 2 + + +@pytest.mark.parametrize( + "provider_kwargs, expected_resource_type", + [ + pytest.param({}, None, id="default_is_self_designed"), + pytest.param({"is_serverless": False}, None, id="self_designed"), + pytest.param({"is_serverless": True}, ["ServerlessCache"], id="serverless"), + ], +) +def test_elasticache_provider_signs_resource_type_only_for_serverless(provider_kwargs, expected_resource_type): + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache-name", + region="us-east-1", + credentials_resolver=_FakeResolver(_FakeCredentials("AKIA-SYNTHETIC")), + **provider_kwargs, + ) + + _, token = provider.get_credentials() + query = parse_qs(urlsplit("https://" + token).query) + + assert query.get("ResourceType") == expected_resource_type + assert query["X-Amz-Signature"] + + +def test_elasticache_provider_lowercases_the_cache_name(): + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="Mixed-Case-Cache", + region="us-east-1", + credentials_resolver=_FakeResolver(_FakeCredentials("AKIA-SYNTHETIC")), + ) + + _, token = provider.get_credentials() + + assert urlsplit("https://" + token).netloc == "mixed-case-cache" + + +def test_elasticache_provider_encodes_reserved_characters_in_the_user_name(): + user_name = "iam user/with+reserved&chars" + provider = ElastiCacheIAMCredentialProvider( + user_name=user_name, + cache_name="cache-name", + region="us-east-1", + credentials_resolver=_FakeResolver(_FakeCredentials("AKIA-SYNTHETIC")), + ) + + returned_user_name, token = provider.get_credentials() + query = parse_qs(urlsplit("https://" + token).query) + + assert returned_user_name == user_name + assert query["User"] == [user_name] + assert query["Action"] == ["connect"] diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9ca0df396b7..0448f8373c1 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -26372,6 +26372,11 @@ export interface components { * @description AWS region for ElastiCache IAM authentication */ aws_iam_region?: string | null; + /** + * Aws Iam Serverless + * @description the ElastiCache cache is serverless rather than a self-designed cluster + */ + aws_iam_serverless?: boolean | string | null; /** * Aws Iam User Name * @description AWS ElastiCache IAM user name From f0e3e031c3fd79acf04db3bca41c4f2f5d1f12ec Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 9 Sep 2026 15:01:06 -0400 Subject: [PATCH 043/126] test(redis): pin ElastiCache IAM signing and TLS coercion invariants Strengthens the serverless test to assert ResourceType is signed rather than merely present, ties _uses_tls to the redis-py kwarg coercion so the two cannot drift, locks the stripped-kwarg name tuple to the test's expectations, and adds "off" and "True" sentinel flag values. Renames the provider builder's parameter to redis_settings. --- tests/test_litellm/test_redis.py | 15 ++++++++++++++- .../test_redis_credential_provider.py | 19 ++++++++++++------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index 5337ed825a7..0e8c86d26df 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -10,13 +10,16 @@ from redis.credentials import CredentialProvider import litellm from litellm._redis import ( + _AWS_IAM_KWARG_NAMES, _async_auth_kwargs, + _coerce_redis_kwargs_types, _get_redis_client_logic, _get_redis_cluster_kwargs, _get_redis_env_kwarg_mapping, _get_redis_kwargs, _get_redis_url_kwargs, _pretty_print_redis_config, + _uses_tls, get_redis_async_client, get_redis_client, get_redis_connection_pool, @@ -31,6 +34,7 @@ from litellm._redis_credential_provider import ( from litellm.caching.redis_cache import RedisCache from litellm.caching.redis_cluster_cache import RedisClusterCache from litellm.constants import REDIS_CLUSTER_HEALTH_CHECK_INTERVAL +from litellm.proxy._types import CoordinationRedisParams class _StubCredentialProvider(CredentialProvider): @@ -127,6 +131,8 @@ def test_aws_iam_settings_are_environment_derived(): mapping = _get_redis_env_kwarg_mapping() assert _AWS_IAM_SETTINGS <= allowed + assert set(_AWS_IAM_KWARG_NAMES) == _AWS_IAM_SETTINGS + assert {f for f in CoordinationRedisParams.model_fields if f.startswith("aws_iam_")} == _AWS_IAM_SETTINGS assert mapping["REDIS_AWS_IAM_AUTH"] == "aws_iam_auth" assert mapping["REDIS_AWS_IAM_USER_NAME"] == "aws_iam_user_name" assert mapping["REDIS_AWS_IAM_CACHE_NAME"] == "aws_iam_cache_name" @@ -348,6 +354,7 @@ def test_aws_iam_environment_settings_install_provider(clean_redis_environment, pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "false"}, id="host_ssl_false_string"), pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "0"}, id="host_ssl_zero_string"), pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "no"}, id="host_ssl_no_string"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "off"}, id="host_ssl_off_string"), pytest.param({"url": "redis://cache.example.com:6379", "ssl": True}, id="plaintext_url"), pytest.param( {"startup_nodes": [{"host": "cache.example.com", "port": 6379}]}, @@ -385,6 +392,7 @@ def test_aws_iam_auth_rejects_non_tls_connections(clean_redis_environment, trans [ pytest.param({"host": "cache.example.com", "port": 6379, "ssl": True}, id="host"), pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "true"}, id="host_ssl_true_string"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "True"}, id="host_ssl_true_capitalized"), pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "1"}, id="host_ssl_one_string"), pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "yes"}, id="host_ssl_yes_string"), pytest.param( @@ -421,6 +429,11 @@ def test_aws_iam_auth_accepts_tls_connections(clean_redis_environment, transport assert isinstance(redis_kwargs["credential_provider"], ElastiCacheIAMCredentialProvider) +@pytest.mark.parametrize("ssl", ["true", "True", "TRUE", "1", "yes", "YES", "false", "0", "no", "off", "", "maybe"]) +def test_tls_detection_agrees_with_the_ssl_kwarg_coercion(ssl): + assert _uses_tls({"ssl": ssl}) is _coerce_redis_kwargs_types({"ssl": ssl})["ssl"] + + def test_aws_iam_settings_are_removed_for_url_and_static_credentials(clean_redis_environment): redis_kwargs = _get_redis_client_logic( url="rediss://url-user:url-pass@cache.example.com:6380", @@ -522,7 +535,7 @@ def test_aws_iam_settings_map_to_distinct_provider_fields(clean_redis_environmen assert provider._region == "iam-region-value" -@pytest.mark.parametrize("aws_iam_auth", [None, False, "", "false", "0", "no"]) +@pytest.mark.parametrize("aws_iam_auth", [None, False, "", "false", "0", "no", "off"]) def test_aws_iam_auth_disabled_does_not_install_provider(clean_redis_environment, aws_iam_auth): redis_kwargs = _get_redis_client_logic( host="cache.example.com", diff --git a/tests/test_litellm/test_redis_credential_provider.py b/tests/test_litellm/test_redis_credential_provider.py index e71fc5d530d..96b1933b0e3 100644 --- a/tests/test_litellm/test_redis_credential_provider.py +++ b/tests/test_litellm/test_redis_credential_provider.py @@ -178,14 +178,14 @@ def test_elasticache_provider_recovers_after_a_failed_resolution(): @pytest.mark.parametrize( - "provider_kwargs, expected_resource_type", + "provider_kwargs, expected_operation_params", [ - pytest.param({}, None, id="default_is_self_designed"), - pytest.param({"is_serverless": False}, None, id="self_designed"), - pytest.param({"is_serverless": True}, ["ServerlessCache"], id="serverless"), + pytest.param({}, frozenset({"Action", "User"}), id="default_is_self_designed"), + pytest.param({"is_serverless": False}, frozenset({"Action", "User"}), id="self_designed"), + pytest.param({"is_serverless": True}, frozenset({"Action", "User", "ResourceType"}), id="serverless"), ], ) -def test_elasticache_provider_signs_resource_type_only_for_serverless(provider_kwargs, expected_resource_type): +def test_elasticache_provider_signs_resource_type_only_for_serverless(provider_kwargs, expected_operation_params): provider = ElastiCacheIAMCredentialProvider( user_name="iam-user", cache_name="cache-name", @@ -195,9 +195,14 @@ def test_elasticache_provider_signs_resource_type_only_for_serverless(provider_k ) _, token = provider.get_credentials() - query = parse_qs(urlsplit("https://" + token).query) + query_string = urlsplit("https://" + token).query + param_names = tuple(pair.split("=", 1)[0] for pair in query_string.split("&")) + first_auth_param = next(i for i, name in enumerate(param_names) if name.startswith("X-Amz-")) + query = parse_qs(query_string) - assert query.get("ResourceType") == expected_resource_type + assert frozenset(param_names[:first_auth_param]) == expected_operation_params + assert all(name.startswith("X-Amz-") for name in param_names[first_auth_param:]) + assert query.get("ResourceType") == (["ServerlessCache"] if "ResourceType" in expected_operation_params else None) assert query["X-Amz-Signature"] From 5d6fec94b74371a7314a4900f3cf025a54f8bcff Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 11:33:21 -0700 Subject: [PATCH 044/126] fix(router): strip Codex harness envelopes before classification --- .../complexity_router/README.md | 8 +- .../complexity_router/complexity_router.py | 9 +- .../complexity_router/config.py | 4 +- .../router_strategy/test_complexity_router.py | 103 +++++++++++++++++- 4 files changed, 118 insertions(+), 6 deletions(-) diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index d605b43e42a..3d2b33a2ecd 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -446,7 +446,11 @@ Reasoning markers in the system prompt do **not** trigger the reasoning override Agent harnesses inject their own context into the conversation as ordinary message text. That text is plumbing, not something a human asked for, so the router strips complete reminder blocks before classifying and picking a tier. A turn that is nothing but a reminder block strips to empty and is skipped, and the router falls back to the last real ask instead -By default a block is anything between `` and ``. `reminder_markers` replaces that with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit: +By default the router strips complete ``, ``, ``, ``, and `` blocks. It also strips Codex repository instructions from the fixed heading prefix `# AGENTS.md instructions for ` through ``, regardless of the repository path + +The Codex `Message Type: NEW_TASK` wrapper and its delegated-task payload remain available for classification. Cleanup applies to the current ask and quoted prior turns; the routed request retains its original content + +`reminder_markers` replaces these defaults with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit: ```yaml model_list: @@ -461,7 +465,7 @@ model_list: close: "[[SUBAGENT_CONTEXT_END]]" ``` -Setting `reminder_markers` replaces the built-in `` pair rather than adding to it, so list that pair too if your harness also emits it. Matching is case-insensitive. Blocks that nest or overlap across pairs are stripped whole. An unclosed delimiter is not a block and is left in place, which keeps prose that merely mentions a delimiter from being eaten +Setting `reminder_markers` replaces all built-in pairs, including the Codex heading pair, so include every default your harness still needs. Matching is case-insensitive. Blocks that nest or overlap across pairs are stripped whole. An unclosed delimiter is not a block and is left in place, which keeps prose that merely mentions a delimiter from being eaten ### Code Detection diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index faafcea404a..923aac85ee2 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -386,7 +386,14 @@ def _effective_turn_off_message_logging(request_kwargs: Mapping[str, object] | N _REMINDER_OPEN: Final = "" _REMINDER_CLOSE: Final = "" -_DEFAULT_REMINDER_MARKERS: Final = ((_REMINDER_OPEN, _REMINDER_CLOSE),) +_DEFAULT_REMINDER_MARKERS: Final = ( + (_REMINDER_OPEN, _REMINDER_CLOSE), + ("", ""), + ("", ""), + ("", ""), + ("", ""), + ("# agents.md instructions for ", ""), +) _TRUNCATION_MARKER: Final = "..." _TRUNCATION_HEAD_FRACTION: Final = 0.3 diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index d28924c69b2..6ab4aa85148 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -1292,8 +1292,8 @@ class ComplexityRouterConfig(BaseModel): "Override the delimiter pairs used to recognize and strip harness-injected reminder " "blocks before classification. A harness that wraps injected context differently per " "agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than " - "adds to, the built-in default of ('', ''), so a " - "harness that also emits that pair lists it too. Matching is case-insensitive." + "adds to, the built-in system-reminder and Codex envelope pairs, so list every " + "built-in pair your harness also emits. Matching is case-insensitive." ), ) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 51103297c58..792963fde23 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -5,6 +5,7 @@ Tests the rule-based complexity scoring and tier assignment logic. """ import asyncio +from copy import deepcopy import logging import sys import time @@ -194,7 +195,14 @@ class TestComplexityRouterInit: complexity_router_config=basic_config, ) - assert router._reminder_markers == (("", ""),) + from litellm.router_strategy.complexity_router.complexity_router import _extract_current_ask_and_system_prompt + + assert ( + _extract_current_ask_and_system_prompt( + [{"role": "user", "content": "noisehello"}], router._reminder_markers + )[0] + == "hello" + ) def test_init_without_config(self, mock_router_instance): """Test initialization without configuration uses defaults.""" @@ -7601,11 +7609,103 @@ _ASKED = {"role": "user", "content": _ASK} _ANSWERED = {"role": "assistant", "content": "Working on it."} _TOOL_RESULT = {"type": "tool_result", "tool_use_id": "x", "content": "out"} _REMINDER = "Budget: 42 tokens remaining. Do not mention this." +_CODEX_NEW_TASK: Final = ( + "Message Type: NEW_TASK\nTask name: /root/cache_worker\nSender: /root\nPayload:\n" + "Implement and test a thread-safe bounded LRU cache." +) +_CODEX_ENVELOPES: Final = ( + "LITELLM ESCALATE cwd=/repo", + "LITELLM ESCALATE plugin list", + "LITELLM ESCALATE preferences", + "LITELLM ESCALATE environment", + "# AGENTS.md instructions for /repo with spaces/中文\nLITELLM ESCALATE instructions", +) class TestContextAwareClassifier: """Test the new classifier context window and trajectory signals.""" + @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) + def test_codex_envelopes_preserve_delegated_task_and_prior_context(self, envelope: str) -> None: + from litellm.router_strategy.complexity_router.complexity_router import ( + _extract_current_ask_and_system_prompt, + _extract_prior_turns, + _newest_turn_ask, + _newest_turn_is_human_ask, + ) + + messages: Final = [ + {"role": "user", "content": f"{envelope}\nDesign cache invalidation"}, + { + "role": "user", + "content": [{"type": "text", "text": envelope}, {"type": "text", "text": _CODEX_NEW_TASK}], + }, + {"role": "developer", "content": "developer scope"}, + {"role": "user", "content": envelope}, + ] + + assert _extract_current_ask_and_system_prompt(messages)[0] == _CODEX_NEW_TASK + assert _extract_prior_turns(messages, _CODEX_NEW_TASK, 1, 100, None, False) == ( + ("user", "Design cache invalidation"), + ) + assert _newest_turn_ask(messages) is None + assert _newest_turn_is_human_ask(messages) is False + assert _extract_current_ask_and_system_prompt([messages[-1]])[0] is None + + @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) + def test_codex_marker_override_and_incomplete_blocks_preserve_text(self, envelope: str) -> None: + from litellm.router_strategy.complexity_router.complexity_router import _strip_reminder_blocks + + incomplete: Final = envelope.rsplit("noise{envelope}", (("", ""),)) == envelope + + @pytest.mark.asyncio + @pytest.mark.parametrize("responses_api", (False, True)) + async def test_codex_routing_preserves_original_request(self, responses_api: bool) -> None: + completion: Final = AsyncMock(return_value=_llm_response('{"tier":"COMPLEX"}')) + router: Final = ComplexityRouter( + model_name="codex-router", + litellm_router_instance=MagicMock(acompletion=completion), + complexity_router_config={ + "tiers": {"COMPLEX": "task-model", "REASONING": "escalated-model"}, + "classifier_type": "llm", + "classifier_llm_config": {"model": "classifier-model"}, + "keyword_tier_rules": [{"keywords": ["LITELLM ESCALATE"], "tier": "REASONING"}], + }, + ) + messages: Final = [ + {"role": "user", "content": _CODEX_NEW_TASK}, + {"role": "user", "content": "\n".join(_CODEX_ENVELOPES)}, + ] + original: Final = deepcopy(messages) + request_kwargs: Final = ( + {"input": messages, "litellm_metadata": {"user_api_key_request_route": "/v1/responses"}} + if responses_api + else {} + ) + + response: Final = await router.async_pre_routing_hook( + model="codex-router", + request_kwargs=request_kwargs, + messages=None if responses_api else messages, + input=messages if responses_api else None, + ) + + assert response is not None + assert response.model == "task-model" + completion.assert_awaited_once() + assert completion.call_args.kwargs["messages"][1]["content"].strip() == ( + f"Classify this message:\n{_CODEX_NEW_TASK}" + ) + assert messages == original + if responses_api: + assert response.messages is None + assert request_kwargs["input"] == original + else: + assert response.messages == original + @pytest.mark.parametrize( "messages,expected_ask", [ @@ -13662,6 +13762,7 @@ class _OutputCeilingRecorder(CustomLogger): def log_pre_api_call(self, model, messages, kwargs): self.seen.append((model, kwargs.get("optional_params", {}).get("max_tokens"))) + NON_REASONING_TIERS: Final = { "NON_REASONING": "gpt-4o-mini", "SIMPLE": "gpt-4o-mini", From 13ddd1ec6481756ac31cac4b7253e23eef3bf656 Mon Sep 17 00:00:00 2001 From: Wolfram Ravenwolf Date: Thu, 10 Sep 2026 20:47:13 +0200 Subject: [PATCH 045/126] fix(wandb): gate reasoning effort on model capabilities --- litellm/llms/wandb/chat/transformation.py | 6 +- ...odel_prices_and_context_window_backup.json | 40 +++++ model_prices_and_context_window.json | 40 +++++ .../wandb/test_wandb_chat_transformation.py | 170 ++++++++++++++---- 4 files changed, 219 insertions(+), 37 deletions(-) diff --git a/litellm/llms/wandb/chat/transformation.py b/litellm/llms/wandb/chat/transformation.py index 9c3ff818331..fdd6644f03d 100644 --- a/litellm/llms/wandb/chat/transformation.py +++ b/litellm/llms/wandb/chat/transformation.py @@ -6,12 +6,16 @@ This is OpenAI compatible - no translation needed / occurs from typing import Final +import litellm from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig class WandbConfig(OpenAIGPTConfig): def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited contract - return super().get_supported_openai_params(model) + ["reasoning_effort"] # mutable-ok: inherited contract + supported_params: Final = super().get_supported_openai_params(model) + if litellm.supports_reasoning(model=model, custom_llm_provider="wandb"): + return supported_params + ["reasoning_effort"] # mutable-ok: inherited contract + return supported_params def map_openai_params( self, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 55618d9f772..da47ef85eb1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45960,6 +45960,7 @@ "output_cost_per_token": 0.0 }, "wandb/openai/gpt-oss-120b": { + "supports_reasoning": true, "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, @@ -45970,6 +45971,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/openai/gpt-oss-20b": { + "supports_reasoning": true, "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, @@ -45980,6 +45982,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/zai-org/GLM-4.5": { + "supports_reasoning": true, "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, @@ -46008,6 +46011,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, @@ -46064,6 +46068,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-V3.1": { + "supports_reasoning": true, "max_tokens": 128000, "max_input_tokens": 161000, "max_output_tokens": 128000, @@ -46074,6 +46079,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-R1-0528": { + "supports_reasoning": true, "max_tokens": 161000, "max_input_tokens": 161000, "max_output_tokens": 161000, @@ -55631,6 +55637,7 @@ "supports_vision": false }, "wandb/deepseek-ai/DeepSeek-V4-Flash": { + "supports_reasoning": true, "max_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.4e-07, @@ -55643,6 +55650,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-V4-Flash-0731": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.3e-07, @@ -55655,6 +55663,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-V4-Pro": { + "supports_reasoning": true, "max_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.15e-06, @@ -55667,6 +55676,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/google/gemma-4-31B-it": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, @@ -55707,6 +55717,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/MiniMaxAI/MiniMax-M3": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.3e-07, @@ -55719,6 +55730,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/moonshotai/Kimi-K2.7-Code": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.1e-07, @@ -55731,6 +55743,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/moonshotai/Kimi-K2.6": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 6.5e-07, @@ -55743,6 +55756,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, @@ -55755,6 +55769,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.5e-07, @@ -55777,6 +55792,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.8-27B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4e-07, @@ -55789,6 +55805,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.6-35B-A3B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -55799,6 +55816,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.6-27B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 6e-07, @@ -55811,6 +55829,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.5-35B-A3B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -55830,7 +55849,28 @@ "supports_vision": false, "source": "https://wandb.ai/site/pricing/tokens/" }, + "wandb/deepseek-ai/DeepSeek-V4-Pro-0813": { + "litellm_provider": "wandb", + "mode": "chat", + "supports_reasoning": true, + "input_cost_per_token": 0.00000131, + "output_cost_per_token": 0.00000396, + "cache_read_input_token_cost": 0.000000044, + "supports_prompt_caching": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/ibm-granite/granite-4.2-8b": { + "litellm_provider": "wandb", + "mode": "chat", + "supports_reasoning": true, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.00000015, + "cache_read_input_token_cost": 0.00000005, + "supports_prompt_caching": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, "wandb/zai-org/GLM-5.2": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.6e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 55618d9f772..da47ef85eb1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45960,6 +45960,7 @@ "output_cost_per_token": 0.0 }, "wandb/openai/gpt-oss-120b": { + "supports_reasoning": true, "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, @@ -45970,6 +45971,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/openai/gpt-oss-20b": { + "supports_reasoning": true, "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, @@ -45980,6 +45982,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/zai-org/GLM-4.5": { + "supports_reasoning": true, "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, @@ -46008,6 +46011,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, @@ -46064,6 +46068,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-V3.1": { + "supports_reasoning": true, "max_tokens": 128000, "max_input_tokens": 161000, "max_output_tokens": 128000, @@ -46074,6 +46079,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-R1-0528": { + "supports_reasoning": true, "max_tokens": 161000, "max_input_tokens": 161000, "max_output_tokens": 161000, @@ -55631,6 +55637,7 @@ "supports_vision": false }, "wandb/deepseek-ai/DeepSeek-V4-Flash": { + "supports_reasoning": true, "max_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.4e-07, @@ -55643,6 +55650,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-V4-Flash-0731": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.3e-07, @@ -55655,6 +55663,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-V4-Pro": { + "supports_reasoning": true, "max_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.15e-06, @@ -55667,6 +55676,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/google/gemma-4-31B-it": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, @@ -55707,6 +55717,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/MiniMaxAI/MiniMax-M3": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.3e-07, @@ -55719,6 +55730,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/moonshotai/Kimi-K2.7-Code": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.1e-07, @@ -55731,6 +55743,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/moonshotai/Kimi-K2.6": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 6.5e-07, @@ -55743,6 +55756,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, @@ -55755,6 +55769,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.5e-07, @@ -55777,6 +55792,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.8-27B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4e-07, @@ -55789,6 +55805,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.6-35B-A3B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -55799,6 +55816,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.6-27B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 6e-07, @@ -55811,6 +55829,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.5-35B-A3B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -55830,7 +55849,28 @@ "supports_vision": false, "source": "https://wandb.ai/site/pricing/tokens/" }, + "wandb/deepseek-ai/DeepSeek-V4-Pro-0813": { + "litellm_provider": "wandb", + "mode": "chat", + "supports_reasoning": true, + "input_cost_per_token": 0.00000131, + "output_cost_per_token": 0.00000396, + "cache_read_input_token_cost": 0.000000044, + "supports_prompt_caching": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/ibm-granite/granite-4.2-8b": { + "litellm_provider": "wandb", + "mode": "chat", + "supports_reasoning": true, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.00000015, + "cache_read_input_token_cost": 0.00000005, + "supports_prompt_caching": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, "wandb/zai-org/GLM-5.2": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.6e-07, diff --git a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py index 4fd66448009..481a52bebad 100644 --- a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py +++ b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py @@ -6,30 +6,90 @@ Nebius AI Studio is an OpenAI-compatible provider with minor customizations. """ import json +from typing import Final import pytest +import respx import litellm from litellm import completion from litellm.llms.wandb.chat.transformation import WandbConfig +WANDB_REASONING_MODELS: Final = ( + "deepseek-ai/DeepSeek-V4-Flash", + "deepseek-ai/DeepSeek-V4-Flash-0731", + "deepseek-ai/DeepSeek-V4-Pro", + "deepseek-ai/DeepSeek-V4-Pro-0813", + "deepseek-ai/DeepSeek-V3.1", + "google/gemma-4-31B-it", + "ibm-granite/granite-4.2-8b", + "MiniMaxAI/MiniMax-M3", + "moonshotai/Kimi-K2.7-Code", + "moonshotai/Kimi-K2.6", + "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B", + "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", + "openai/gpt-oss-120b", + "openai/gpt-oss-20b", + "Qwen/Qwen3.8-27B", + "Qwen/Qwen3.6-35B-A3B", + "Qwen/Qwen3.6-27B", + "Qwen/Qwen3.5-35B-A3B", + "zai-org/GLM-5.2", + "moonshotai/Kimi-K2.5", + "MiniMaxAI/MiniMax-M2.5", + "zai-org/GLM-4.5", + "Qwen/Qwen3-235B-A22B-Thinking-2507", + "deepseek-ai/DeepSeek-R1-0528", +) + + +@pytest.fixture +def wandb_test_config(local_model_cost_map, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "telemetry", False) + monkeypatch.setattr(litellm, "drop_params", False) + + +@pytest.fixture +def wandb_request_mock(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post("https://api.inference.wandb.ai/v1/chat/completions").respond( + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Done"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + status_code=200, + ) + + class TestWandbConfig: """Test class for WandB Inference functionality""" - def test_map_openai_params_preserves_reasoning_effort(self): - supported_params = litellm.get_supported_openai_params(model="wandb/openai/gpt-oss-20b") + @pytest.mark.parametrize("model", WANDB_REASONING_MODELS) + def test_map_openai_params_preserves_reasoning_effort(self, wandb_test_config, model: str): + assert litellm.model_cost[f"wandb/{model}"].get("supports_reasoning") is True + supported_params = litellm.get_supported_openai_params(model=f"wandb/{model}") assert supported_params is not None assert "reasoning_effort" in supported_params result = WandbConfig().map_openai_params( - non_default_params={"reasoning_effort": "medium"}, + non_default_params={"reasoning_effort": "medium", "max_completion_tokens": 64}, optional_params={}, - model="openai/gpt-oss-20b", + model=model, drop_params=True, ) - assert result == {"reasoning_effort": "medium"} + assert result == {"reasoning_effort": "medium", "max_tokens": 64} def test_default_api_base(self): """Test that default API base is used when none is provided""" @@ -155,40 +215,78 @@ class TestWandbConfig: assert "Hey from LiteLLM" in content @pytest.mark.respx() - def test_wandb_completion_preserves_reasoning_effort_with_drop_params(self, respx_mock, monkeypatch): - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - - api_base = "https://api.inference.wandb.ai/v1" - request_mock = respx_mock.post(f"{api_base}/chat/completions").respond( - json={ - "id": "chatcmpl-123", - "object": "chat.completion", - "created": 1677652288, - "model": "gpt-oss-20b", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "Done"}, - "finish_reason": "stop", - } - ], - "usage": { - "prompt_tokens": 1, - "completion_tokens": 1, - "total_tokens": 2, - }, - }, - status_code=200, - ) - + @pytest.mark.parametrize( + "model,effort", + tuple((model, "medium") for model in WANDB_REASONING_MODELS) + + ( + ("Qwen/Qwen3.8-27B", "low"), + ("Qwen/Qwen3.8-27B", "xhigh"), + ), + ) + def test_wandb_completion_preserves_reasoning_effort_with_drop_params( + self, wandb_test_config, wandb_request_mock: respx.Route, model: str, effort: str + ): completion( - model="wandb/openai/gpt-oss-20b", + model=f"wandb/{model}", messages=[{"role": "user", "content": "Hello"}], api_key="fake-wandb-key", - api_base=api_base, - reasoning_effort="medium", + api_base="https://api.inference.wandb.ai/v1", + reasoning_effort=effort, + max_completion_tokens=64, drop_params=True, ) - request_body = json.loads(request_mock.calls[0].request.content) - assert request_body["reasoning_effort"] == "medium" + assert wandb_request_mock.call_count == 1 + request_body = json.loads(wandb_request_mock.calls[0].request.content) + assert request_body["model"] == model + assert request_body["reasoning_effort"] == effort + assert request_body["max_tokens"] == 64 + assert "max_completion_tokens" not in request_body + + @pytest.mark.respx(assert_all_called=False) + @pytest.mark.parametrize("drop_params", [True, False]) + @pytest.mark.parametrize( + "model,explicit_false", + [ + ("meta-llama/Llama-3.1-8B-Instruct", False), + ("unknown-model", False), + ("openai/gpt-oss-20b", True), + ], + ) + def test_wandb_completion_without_reasoning_support( + self, + wandb_test_config, + wandb_request_mock: respx.Route, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + model: str, + explicit_false: bool, + drop_params: bool, + ): + with monkeypatch.context() as context: + if explicit_false: + context.setitem(litellm.model_cost[f"wandb/{model}"], "supports_reasoning", False) + + kwargs = { + "model": f"wandb/{model}", + "messages": [{"role": "user", "content": "Hello"}], + "api_key": "fake-wandb-key", + "api_base": "https://api.inference.wandb.ai/v1", + "reasoning_effort": "medium", + "drop_params": drop_params, + } + if not drop_params: + with pytest.raises(litellm.UnsupportedParamsError, match="reasoning_effort"): + completion(**kwargs) + assert len(respx_mock.calls) == 0 + return + + completion(**kwargs) + assert wandb_request_mock.call_count == 1 + request_body = json.loads(wandb_request_mock.calls[0].request.content) + assert request_body["model"] == model + assert "reasoning_effort" not in request_body + + supported_params = litellm.get_supported_openai_params(model=f"wandb/{model}") + assert supported_params is not None + assert "reasoning_effort" not in supported_params From 153c962e15aa4a9506abc8dd7989b9d05f864faf Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 11:49:57 -0700 Subject: [PATCH 046/126] feat(router): log classifier input and masked source request --- litellm/integrations/custom_logger.py | 11 ++ .../litellm_core_utils/classifier_logging.py | 59 ++++++++ litellm/litellm_core_utils/litellm_logging.py | 27 +++- litellm/litellm_core_utils/redact_messages.py | 10 ++ litellm/llms/openai/openai.py | 2 + .../spend_management_endpoints.py | 20 ++- .../spend_tracking/spend_tracking_utils.py | 5 + .../complexity_router/complexity_router.py | 11 +- litellm/types/utils.py | 8 +- tests/e2e/coverage_registry/reliability.yaml | 1 + tests/e2e/router/test_classifier_audit_e2e.py | 143 ++++++++++++++++++ .../test_classifier_logging.py | 38 +++++ .../test_litellm_logging.py | 121 ++++++++++++++- .../test_redact_messages.py | 24 +++ .../test_spend_management_endpoints.py | 23 +++ .../test_spend_tracking_utils.py | 24 +++ .../router_strategy/test_complexity_router.py | 23 +++ .../ClassifierAuditView.test.tsx | 56 +++++++ .../LogDetailsDrawer/ClassifierAuditView.tsx | 52 +++++++ .../LogDetailsDrawer/LogDetailContent.tsx | 53 +++---- 20 files changed, 673 insertions(+), 38 deletions(-) create mode 100644 litellm/litellm_core_utils/classifier_logging.py create mode 100644 tests/e2e/router/test_classifier_audit_e2e.py create mode 100644 tests/test_litellm/litellm_core_utils/test_classifier_logging.py create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.tsx diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 8f03e08f02d..473bf223105 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -927,6 +927,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac # Handle turn_off_message_logging - redact messages and responses (if not already excluded) if turn_off_message_logging: + from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit + + for field in CLASSIFIER_AUDIT_FIELDS: + standard_logging_object_copy.pop(field, None) + params: Final = model_call_details_copy.get("litellm_params") + request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None + if isinstance(params, dict) and isinstance(request, dict): + model_call_details_copy["litellm_params"] = { + **params, + "proxy_server_request": without_classifier_audit(request), + } redacted_str: Final = "redacted-by-litellm" if "messages" not in (excluded_fields or []) and standard_logging_object_copy.get("messages") is not None: diff --git a/litellm/litellm_core_utils/classifier_logging.py b/litellm/litellm_core_utils/classifier_logging.py new file mode 100644 index 00000000000..b9cb52a190d --- /dev/null +++ b/litellm/litellm_core_utils/classifier_logging.py @@ -0,0 +1,59 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import JsonValue, TypeAdapter, ValidationError + +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload +from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ClassifierAudit + +CLASSIFIER_AUDIT_FIELDS: Final = ("classifier_input", "originating_request_masked") +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def classifier_input_snapshot(value: object, *, openai_sdk: bool = False) -> Mapping[str, JsonValue] | None: + if openai_sdk and isinstance(value, Mapping): + body: Final = { + key: item for key, item in value.items() if key not in ("extra_headers", "extra_query", "extra_body") + } + extra_body: Final = value.get("extra_body") + return classifier_input_snapshot({**body, **extra_body} if isinstance(extra_body, Mapping) else body) + try: + return ( + _JSON_OBJECT.validate_json(value) + if isinstance(value, (str, bytes)) + else _JSON_OBJECT.validate_python(value) + ) + except ValidationError: + return None + + +def is_classifier_call(call_type: str, params: Mapping[str, object]) -> bool: + return call_type in ("completion", "acompletion") and any( + isinstance(metadata := params.get(key), Mapping) + and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == AUTOROUTER_CLASSIFIER_CALL_ORIGIN + for key in ("metadata", "litellm_metadata") + ) + + +def masked_originating_request(request_kwargs: Mapping[str, object] | None) -> Mapping[str, JsonValue] | None: + request: Final = (request_kwargs or {}).get("proxy_server_request") + body: Final = request.get("body") if isinstance(request, Mapping) else None + if not isinstance(body, Mapping): + return None + serializable: Final = classifier_input_snapshot(safe_dumps(body)) + return classifier_input_snapshot(redact_credentials_in_payload(serializable)) if serializable is not None else None + + +def classifier_audit_fields(payload: Mapping[str, object]) -> ClassifierAudit: + classifier_input: Final = classifier_input_snapshot(payload.get("classifier_input")) + originating_request: Final = classifier_input_snapshot(payload.get("originating_request_masked")) + return { + **(ClassifierAudit(classifier_input=classifier_input) if classifier_input is not None else {}), + **(ClassifierAudit(originating_request_masked=originating_request) if originating_request is not None else {}), + } + + +def without_classifier_audit(payload: Mapping[str, object]) -> dict[str, object]: + return {key: value for key, value in payload.items() if key not in CLASSIFIER_AUDIT_FIELDS} diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index cb9209be267..f85aa32829f 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -17,7 +17,7 @@ from types import MappingProxyType, TracebackType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast from httpx import Response -from pydantic import BaseModel +from pydantic import BaseModel, JsonValue import litellm from litellm import ( @@ -64,6 +64,11 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.deepeval.deepeval import DeepEvalLogger from litellm.integrations.mlflow import MlflowLogger from litellm.integrations.sqs import SQSLogger +from litellm.litellm_core_utils.classifier_logging import ( + classifier_audit_fields, + classifier_input_snapshot, + is_classifier_call, +) from litellm.litellm_core_utils.core_helpers import is_expected_client_error, reconstruct_model_name from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.internal_call_metadata import ( @@ -89,6 +94,7 @@ from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_custom_logger, redact_message_input_output_from_logging, redact_streaming_responses_for_custom_logger, + should_redact_message_logging, ) from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse @@ -473,6 +479,7 @@ class Logging(LiteLLMLoggingBaseClass): stream_options = None litellm_request_debug: bool = False streamed_anthropic_message_id: str | None = None + classifier_input: Mapping[str, JsonValue] | None = None def __init__( self, @@ -1211,6 +1218,14 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["api_key"] = api_key self.model_call_details["additional_args"] = additional_args self.model_call_details["log_event_type"] = "pre_api_call" + if is_classifier_call(self.call_type, self.model_call_details.get("litellm_params") or {}): + self.classifier_input = ( + None + if should_redact_message_logging(self.model_call_details) + else classifier_input_snapshot( + additional_args.get("complete_input_dict"), openai_sdk=additional_args.get("openai_sdk") is True + ) + ) if model: # if model name was changes pre-call, overwrite the initial model call name with the new one self.model_call_details["model"] = model self.model_call_details["litellm_params"]["api_base"] = self._get_masked_api_base( @@ -6293,6 +6308,16 @@ def get_standard_logging_object_payload( ) payload: Final[StandardLoggingPayload] = StandardLoggingPayload( + **( + classifier_audit_fields( + { + "classifier_input": logging_obj.classifier_input, + "originating_request_masked": proxy_server_request.get("originating_request_masked"), + } + ) + if is_classifier_call(call_type or "", litellm_params) and not should_redact_message_logging(kwargs) + else {} + ), id=str(id), litellm_call_id=kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"), trace_id=StandardLoggingPayloadSetup.get_standard_logging_payload_trace_id( diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index a567854d767..c97566c21a8 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Any, Final import litellm from litellm.constants import REDACTED_BY_LITELLM from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, ) @@ -183,6 +184,9 @@ def _redact_standard_logging_object(model_call_details: dict): redacted_str: Final = REDACTED_BY_LITELLM + for field in CLASSIFIER_AUDIT_FIELDS: + standard_logging_object.pop(field, None) + if standard_logging_object.get("messages") is not None: standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}] @@ -254,6 +258,12 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons copy via redact_streaming_responses_for_custom_logger instead. """ # Redact model_call_details + for field in CLASSIFIER_AUDIT_FIELDS: + model_call_details.pop(field, None) + params: Final = model_call_details.get("litellm_params") + request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None + if isinstance(params, dict) and isinstance(request, Mapping): + model_call_details["litellm_params"] = {**params, "proxy_server_request": without_classifier_audit(request)} model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}] model_call_details["prompt"] = "" model_call_details["input"] = "" diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index edc8d64d9c2..5f04ebe0c01 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -797,6 +797,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): "api_base": openai_client._base_url._uri_reference, "acompletion": acompletion, "complete_input_dict": data, + "openai_sdk": True, }, ) @@ -938,6 +939,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): "api_base": openai_aclient._base_url._uri_reference, "acompletion": True, "complete_input_dict": data, + "openai_sdk": True, }, ) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index f8831ca4152..ee43972309b 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -26,7 +26,8 @@ from typing_extensions import ReadOnly import litellm from litellm._logging import verbose_proxy_logger -from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME +from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME +from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields, classifier_input_snapshot from litellm.proxy._types import * from litellm.proxy._types import ProviderBudgetResponse, ProviderBudgetResponseObject from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -3099,7 +3100,11 @@ async def _resolve_request_response_payload( proxy_server_request: Final = row.get("proxy_server_request") pg_payload: Final = RequestResponsePayload(messages, response, proxy_server_request) - if ( + stored_request: Final = classifier_input_snapshot(proxy_server_request) + truncated_audit: Final = bool(stored_request and classifier_audit_fields(stored_request)) and ( + LITELLM_TRUNCATED_PAYLOAD_FIELD in str(proxy_server_request) + ) + if not truncated_audit and ( _spend_log_field_has_content(messages) or _spend_log_field_has_content(response) or _spend_log_field_has_content(proxy_server_request) @@ -3124,10 +3129,19 @@ async def _resolve_request_response_payload( if payload is None: return pg_payload + cold_audit: Final = classifier_audit_fields(payload) + resolved_request: Final = ( + {**(classifier_input_snapshot(payload.get("proxy_server_request")) or stored_request or {}), **cold_audit} + if cold_audit + else payload.get("proxy_server_request") + ) + if truncated_audit: + return RequestResponsePayload(messages, response, resolved_request if cold_audit else proxy_server_request) + return RequestResponsePayload( messages=payload.get("messages"), response=payload.get("response"), - proxy_server_request=payload.get("proxy_server_request"), + proxy_server_request=resolved_request, ) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index a21d761996f..da55d1cb153 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -22,6 +22,7 @@ from litellm.constants import ( from litellm.constants import ( MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB, ) +from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, reconstruct_model_name, @@ -1251,6 +1252,10 @@ def _get_proxy_server_request_for_spend_logs_payload( if _proxy_server_request is not None: _request_body = _proxy_server_request.get("body", {}) or {} + standard_payload: Final = (kwargs or {}).get("standard_logging_object") + if isinstance(standard_payload, Mapping): + _request_body = {**_request_body, **classifier_audit_fields(standard_payload)} + if kwargs is not None: realtime_tools: Final = kwargs.get("realtime_tools") if realtime_tools: diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index faafcea404a..162fcf2efb0 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -36,6 +36,7 @@ from litellm.constants import ( SESSION_ID_GENERATED_METADATA_KEY, ) from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.classifier_logging import masked_originating_request from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, get_metadata_variable_name_from_kwargs, @@ -1924,11 +1925,8 @@ class ComplexityRouter(CustomLogger): Call the configured classifier model with a system/user role split and prior-turn context. Builds a structured classification prompt with: - - System message: the stable classifier rubric AND the caller's own system prompt (task - constraints). This is the largest, most repeated part of the call, so keeping it in the - system role lets the provider prompt-cache it across a session's classifier calls. - - User message: the variable payload -- a few prior user turns for context and the current - ask to classify. + - System message: the stable classifier rubric. + - User message: the caller's system prompt quoted as task context, prior turns, and the current ask. Args: prompt: The current user ask text (already extracted as the real human ask, not tool results) @@ -2005,12 +2003,13 @@ class ComplexityRouter(CustomLogger): classifier_call_params = MappingProxyType({"reasoning_effort": llm_config.reasoning_effort}) proxy_server_request: Final = { + "originating_request_masked": masked_originating_request(request_kwargs), "body": { "model": llm_config.model, "messages": messages_for_call, "response_format": response_format, **classifier_call_params, - } + }, } classifier_timeout_s: Final[float] = llm_config.timeout_ms / 1000 diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ab0cc5f959c..58b940227f8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -35,6 +35,7 @@ from pydantic import ( BaseModel, ConfigDict, Field, + JsonValue, PrivateAttr, SkipValidation, field_serializer, @@ -3373,7 +3374,12 @@ class StandardAuditLogPayload(TypedDict): updated_values: str | None -class StandardLoggingPayload(TypedDict): +class ClassifierAudit(TypedDict, total=False): + classifier_input: ReadOnly[Mapping[str, JsonValue]] + originating_request_masked: ReadOnly[Mapping[str, JsonValue]] + + +class StandardLoggingPayload(ClassifierAudit): id: str trace_id: str # Trace multiple LLM calls belonging to same overall request (e.g. fallbacks/retries) session_id: str # End-user/conversation session id (litellm_session_id), independent of trace_id diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 6b69677d490..4d40f19e67b 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -1,4 +1,5 @@ # Reliability & Performance (behavior features). Grounded in litellm/router.py + router_strategy/ + router_utils/. +- {id: reliability.routing.classifier_audit.separates_provider_input_and_source, module: reliability, tier: P1, behavior: routing, variant: classifier_audit, assertions: [separates_provider_input_and_source], exercised_on: [chat_completions, messages, responses], source: "litellm/litellm_core_utils/classifier_logging.py", rationale: "Classifier spend details distinguish provider input from the credential-masked originating request across all three request surfaces"} - {id: reliability.fallback.5xx.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: "5xx", assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:2024", rationale: "Reroute on provider 5xx to alternate deployment"} - {id: reliability.fallback.context_window.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: context_window, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6108", rationale: "Fallback when model exceeds context limit"} - {id: reliability.fallback.content_policy.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: content_policy, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6023", rationale: "Reroute on content-policy violation"} diff --git a/tests/e2e/router/test_classifier_audit_e2e.py b/tests/e2e/router/test_classifier_audit_e2e.py new file mode 100644 index 00000000000..6b444441477 --- /dev/null +++ b/tests/e2e/router/test_classifier_audit_e2e.py @@ -0,0 +1,143 @@ +import json +import os +from collections.abc import Iterator +from contextlib import ExitStack +from dataclasses import dataclass +from typing import Final, Literal + +import pytest +from pydantic import BaseModel, Field, JsonValue, TypeAdapter + +from e2e_config import unique_marker +from e2e_http import AnthropicHeaders, NoBody, unwrap +from models import ChatMessage, KeyGenerateBody, LiteLLMParamsBody +from proxy_client import ProxyClient + +pytestmark = pytest.mark.e2e + + +class AuditMetadata(BaseModel): + source_marker: str + authorization: str = "synthetic-audit-secret" + + +class AuditHeaders(AnthropicHeaders): + enable_redaction: str | None = Field(default=None, serialization_alias="x-litellm-enable-message-redaction") + + +class AuditRequest(BaseModel): + model: str + messages: list[ChatMessage] | None = None + system: str | None = None + instructions: str | None = None + input: str | None = None + max_tokens: int | None = None + max_output_tokens: int | None = None + metadata: AuditMetadata | None = None + litellm_metadata: AuditMetadata | None = None + + +class AuditResponse(BaseModel): + id: str + + +class AuditDetail(BaseModel): + proxy_server_request: dict[str, JsonValue] | str | None = None + response: dict[str, JsonValue] | str | None = None + + +@dataclass(frozen=True, slots=True) +class AuditDeployment: + alias: str + key: str + + +@pytest.fixture +def audit_deployment(proxy: ProxyClient, provider: str) -> Iterator[AuditDeployment]: + marker: Final = unique_marker() + classifier: Final = f"audit-classifier-{marker}" + target: Final = f"audit-target-{marker}" + alias: Final = f"audit-router-{marker}" + model: Final = os.environ.get( + f"E2E_CHEAP_{provider.upper()}_MODEL", "gpt-5.6" if provider == "openai" else "claude-haiku-4-5" + ) + params: Final = LiteLLMParamsBody( + model=f"{provider}/{model}", + api_key=os.environ.get(f"{provider.upper()}_API_KEY") or f"os.environ/{provider.upper()}_API_KEY", + api_base=os.environ.get(f"{provider.upper()}_API_BASE"), + ) + with ExitStack() as stack: + for name in (classifier, target): + stack.callback(proxy.delete_model, proxy.create_model(name, params)) + stack.callback(proxy.delete_model, proxy.create_model(alias, LiteLLMParamsBody( + model="auto_router/complexity_router", + complexity_router_config={ + "classifier_type": "llm", + "classifier_llm_config": {"model": classifier, "timeout_ms": 30000}, + "tiers": {tier: target for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")}, + }, + ))) + key: Final = proxy.generate_key(KeyGenerateBody(models=[alias, classifier, target])) + stack.callback(proxy.delete_key, key) + yield AuditDeployment(alias, key) + + +class TestClassifierAudit: + @pytest.mark.covers( + "reliability.routing.classifier_audit.separates_provider_input_and_source", + exercised_on=("chat_completions", "messages", "responses"), + ) + @pytest.mark.parametrize("surface", ["chat_completions", "messages", "responses"]) + @pytest.mark.parametrize("provider", ["openai", "anthropic"]) + @pytest.mark.parametrize("redact", [False, True]) + def test_classifier_audit_separates_input_and_masked_source( + self, proxy: ProxyClient, audit_deployment: AuditDeployment, surface: Literal["chat_completions", "messages", "responses"], + redact: bool, + ) -> None: + marker: Final = unique_marker() + source_marker: Final = f"source-only-{marker}" + prompt: Final = f"Reply with hello. Request label {marker}" + metadata: Final = AuditMetadata(source_marker=source_marker) + body: Final = AuditRequest( + model=audit_deployment.alias, + messages=[ChatMessage(role="user", content=prompt)] if surface != "responses" else None, + input=prompt if surface == "responses" else None, + system="Be concise" if surface == "messages" else None, + instructions="Be concise" if surface == "responses" else None, + max_tokens=128 if surface != "responses" else None, + max_output_tokens=128 if surface == "responses" else None, + metadata=metadata if surface != "responses" else None, + litellm_metadata=metadata if surface == "responses" else None, + ) + path: Final = {"chat_completions": "/chat/completions", "messages": "/v1/messages", "responses": "/v1/responses"}[surface] + response: Final = unwrap(proxy.transport.post( + path, headers=AuditHeaders(authorization=f"Bearer {audit_deployment.key}", enable_redaction="true" if redact else None), + json=body, response_type=AuditResponse, + )) + assert response.id + rows: Final = proxy.poll_logs_for_key(audit_deployment.key, min_rows=2) + assert len(rows) == 2, "Expected a classifier spend row and a routed response spend row" + details: Final = tuple( + unwrap(proxy.transport.get( + f"/spend/logs/ui/{row.request_id}", headers=proxy.transport.master, + params=NoBody(), response_type=AuditDetail, + )) for row in rows + ) + adapter: Final = TypeAdapter(dict[str, JsonValue]) + requests: Final = tuple( + adapter.validate_json(detail.proxy_server_request) if isinstance(detail.proxy_server_request, str) + else detail.proxy_server_request or {} for detail in details + ) + audits: Final = tuple(item for item in requests if "classifier_input" in item) + if redact: + assert audits == () + assert all("originating_request_masked" not in item for item in requests) + return + assert len(audits) == 1, "The audit belongs only to the classifier call" + audit: Final = audits[0] + assert marker in json.dumps(audit["classifier_input"]) + assert source_marker not in json.dumps(audit["classifier_input"]) + assert source_marker in json.dumps(audit["originating_request_masked"]) + assert "synthetic-audit-secret" not in json.dumps(audit) + assert audit_deployment.alias in json.dumps(audit["originating_request_masked"]) + assert any("tier" in str(detail.response) for detail in details) diff --git a/tests/test_litellm/litellm_core_utils/test_classifier_logging.py b/tests/test_litellm/litellm_core_utils/test_classifier_logging.py new file mode 100644 index 00000000000..c923225c750 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_classifier_logging.py @@ -0,0 +1,38 @@ +from typing import Final + +import pytest + +from litellm.litellm_core_utils.classifier_logging import classifier_input_snapshot, masked_originating_request + + +@pytest.mark.parametrize("encoded", [False, True]) +def test_classifier_snapshot_preserves_provider_shape_and_is_independent(encoded: bool) -> None: + import json + + provider_body: Final = {"system": [{"text": "rubric"}], "messages": [{"role": "user", "content": "ask"}]} + snapshot: Final = classifier_input_snapshot(json.dumps(provider_body) if encoded else provider_body) + assert snapshot == provider_body + provider_body["messages"][0]["content"] = "later mutation" + assert snapshot == {"system": [{"text": "rubric"}], "messages": [{"role": "user", "content": "ask"}]} + + +def test_originating_snapshot_masks_nested_credentials_without_altering_source() -> None: + body: Final = { + "model": "router", + "input": [{"type": "message", "role": "user", "content": "source-only"}], + "api_key": "short", + "metadata": {"nested": [{"Authorization": "Bearer secret", "access_token": 123}]}, + } + snapshot: Final = masked_originating_request({"proxy_server_request": {"body": body}}) + assert snapshot is not None + assert snapshot["model"] == "router" + assert snapshot["input"] == body["input"] + assert snapshot["api_key"] == "REDACTED" + assert snapshot["metadata"] == {"nested": [{"Authorization": "REDACTED", "access_token": "REDACTED"}]} + assert body["api_key"] == "short" + assert body["metadata"]["nested"][0]["Authorization"] == "Bearer secret" + + +@pytest.mark.parametrize("value", [None, "not-json", [], {"messages": object()}]) +def test_invalid_provider_payload_is_not_reported_as_captured(value: object) -> None: + assert classifier_input_snapshot(value) is None diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 6aa77745e3d..28545bb2349 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -3,7 +3,7 @@ import contextlib import datetime import os import sys -from typing import Literal +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -6801,3 +6801,122 @@ def test_get_error_information_redacts_provider_key_from_upstream_url(): assert "REDACTED" in result["traceback"] assert "REDACTED" in result["error_message"] assert result["error_code"] == "400" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider", ["openai", "anthropic", "bedrock"]) +async def test_classifier_audit_matches_provider_transport(provider: str) -> None: + import json + + from openai import AsyncOpenAI + + from litellm.litellm_core_utils.classifier_logging import classifier_input_snapshot + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + outbound: Final = asyncio.Queue() + logs: Final = asyncio.Queue() + + def respond(request: httpx.Request) -> httpx.Response: + outbound.put_nowait(json.loads(request.content)) + content: Final = '{"tier":"SIMPLE"}' + if provider == "anthropic": + return httpx.Response(200, json={ + "id": "msg-audit", "type": "message", "role": "assistant", "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": content}], "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }) + if provider == "bedrock": + return httpx.Response(200, json={ + "output": {"message": {"role": "assistant", "content": [{"text": content}]}}, + "stopReason": "end_turn", "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15}, + "metrics": {"latencyMs": 1}, + }) + return httpx.Response(200, json={ + "id": "chatcmpl-audit", "object": "chat.completion", "created": 0, "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }) + + async def capture(kwargs, response_obj, start_time, end_time): + logs.put_nowait(kwargs["standard_logging_object"]) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + handler: Final = AsyncHTTPHandler() + await handler.close() + handler.client = http_client + client: Final = AsyncOpenAI(api_key="transport-only", http_client=http_client) if provider == "openai" else handler + model: Final = { + "openai": "openai/gpt-5.6", + "anthropic": "anthropic/claude-haiku-4-5", + "bedrock": "bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", + }[provider] + + async def run(marker: str) -> None: + await litellm.acompletion( + model=model, api_key="transport-only", client=client, max_tokens=128, + aws_access_key_id="transport-only", aws_secret_access_key="transport-only", aws_region_name="us-east-1", + messages=[{"role": "system", "content": "classifier-rubric"}, {"role": "user", "content": marker}], + metadata={"internal_call_origin": "autorouter_classifier"}, + proxy_server_request={"body": {}, "originating_request_masked": {"input": f"source-only-{marker}"}}, + success_callback=[capture], num_retries=0, + **({"extra_body": {"audit_context": "provider-extra"}, "extra_headers": {"X-Audit": "header-only-secret"}} + if provider == "openai" else {}), + ) + + await asyncio.gather(run("request-one"), run("request-two")) + requests: Final = await asyncio.wait_for(asyncio.gather(outbound.get(), outbound.get()), timeout=10) + payloads: Final = await asyncio.wait_for(asyncio.gather(logs.get(), logs.get()), timeout=10) + for payload in payloads: + snapshot: Final = payload["classifier_input"] + assert snapshot in requests + assert "source-only" not in json.dumps(snapshot) + assert "classifier-rubric" in json.dumps(snapshot) + assert "transport-only" not in json.dumps(snapshot) + assert "header-only-secret" not in json.dumps(snapshot) + assert "SIMPLE" in json.dumps(payload["response"]) + marker: Final = "request-one" if "request-one" in json.dumps(snapshot) else "request-two" + assert payload["originating_request_masked"] == {"input": f"source-only-{marker}"} + assert classifier_input_snapshot(snapshot) is not None + if provider != "openai": + assert all("system" in request for request in requests) + + +@pytest.mark.parametrize("redaction", ["none", "global", "request", "header"]) +@pytest.mark.parametrize("status", ["success", "failure"]) +def test_classifier_audit_obeys_message_logging_before_payload_emission(logging_obj, monkeypatch, redaction, status): + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + + monkeypatch.setattr(litellm, "turn_off_message_logging", redaction == "global") + params: Final = { + "metadata": {"internal_call_origin": "autorouter_classifier", **( + {"headers": {"x-litellm-enable-message-redaction": "true"}} if redaction == "header" else {} + )}, + "proxy_server_request": {"body": {}, "originating_request_masked": {"input": "source-only"}}, + } + logging_obj.model_call_details["litellm_params"] = params + logging_obj.model_call_details["standard_callback_dynamic_params"] = ( + {"turn_off_message_logging": True} if redaction == "request" else {} + ) + logging_obj.pre_call( + input=[], api_key=None, additional_args={"complete_input_dict": {"system": "rubric", "messages": []}} + ) + now: Final = datetime.datetime.now() + payload: Final = get_standard_logging_object_payload( + kwargs={**logging_obj.model_call_details, "call_type": "completion"}, init_response_obj={}, + start_time=now, end_time=now, logging_obj=logging_obj, status=status, + ) + assert payload is not None + if redaction == "none": + assert payload["classifier_input"] == {"system": "rubric", "messages": []} + assert payload["originating_request_masked"] == {"input": "source-only"} + else: + assert "classifier_input" not in payload + assert "originating_request_masked" not in payload + + +@pytest.mark.parametrize("call_type,origin", [("completion", None), ("aembedding", "autorouter_classifier")]) +def test_classifier_audit_is_not_added_to_other_calls(logging_obj, call_type, origin): + logging_obj.call_type = call_type + logging_obj.model_call_details["litellm_params"] = {"metadata": {"internal_call_origin": origin}} + logging_obj.pre_call(input=[], api_key=None, additional_args={"complete_input_dict": {"input": "embedding"}}) + assert logging_obj.classifier_input is None diff --git a/tests/test_litellm/litellm_core_utils/test_redact_messages.py b/tests/test_litellm/litellm_core_utils/test_redact_messages.py index 3be0bae4120..1ba9901aab0 100644 --- a/tests/test_litellm/litellm_core_utils/test_redact_messages.py +++ b/tests/test_litellm/litellm_core_utils/test_redact_messages.py @@ -6,6 +6,7 @@ but litellm_params["litellm_metadata"] is None. """ import threading +from typing import Final from types import SimpleNamespace import pytest @@ -776,6 +777,29 @@ class TestPerformRedaction: assert response_obj.choices[0].message.content == "secret content" + +@pytest.mark.parametrize("callback_only", [False, True]) +def test_classifier_audit_redaction_removes_both_fields_and_source_carrier(callback_only: bool) -> None: + audit: Final = {"classifier_input": {"system": "private rubric"}, "originating_request_masked": {"input": "private source"}} + details: Final = { + "standard_logging_object": {**audit, "messages": [], "response": {}}, + "litellm_params": {"proxy_server_request": {"body": {}, "originating_request_masked": audit["originating_request_masked"]}}, + } + logger: Final = CustomLogger() + logger.turn_off_message_logging = True + if callback_only: + redacted: Final = logger.redact_standard_logging_payload_from_model_call_details(details) + assert "classifier_input" not in redacted["standard_logging_object"] + assert "originating_request_masked" not in redacted["standard_logging_object"] + assert "originating_request_masked" not in redacted["litellm_params"]["proxy_server_request"] + assert details["standard_logging_object"]["classifier_input"] == audit["classifier_input"] + assert details["litellm_params"]["proxy_server_request"]["originating_request_masked"] == audit["originating_request_masked"] + else: + perform_redaction(details, result=None) + assert "classifier_input" not in details["standard_logging_object"] + assert "originating_request_masked" not in details["standard_logging_object"] + assert "originating_request_masked" not in details["litellm_params"]["proxy_server_request"] + def test_unredactable_result_is_not_deepcopied(self): """A result shape no branch can redact must not be deepcopied. diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 329a33eb440..a65209b1456 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -5703,6 +5703,29 @@ def _cold_storage_handler(payload): return ColdStorageHandler(cold_storage_logger=logger), logger +@pytest.mark.asyncio +@pytest.mark.parametrize("cold_has_audit", [False, True]) +async def test_resolve_payload_recovers_truncated_classifier_audit_without_losing_existing_fields(cold_has_audit): + full_audit = {"classifier_input": {"system": "full rubric"}, "originating_request_masked": {"input": "source"}} + truncated_request = {"model": "classifier", "classifier_input": {"system": "litellm_truncated"}} + handler, logger = _cold_storage_handler({ + "proxy_server_request": {"body": {}}, **(full_audit if cold_has_audit else {}), + }) + row = { + "messages": '[{"role":"user","content":"ask"}]', "response": '{"tier":"SIMPLE"}', + "proxy_server_request": json.dumps(truncated_request), "metadata": {"cold_storage_object_key": "k/audit.json"}, + } + resolved = await spend_management_endpoints._resolve_request_response_payload(row, cold_storage_handler=handler) + assert logger.requested_object_keys == ["k/audit.json"] + assert resolved.messages == row["messages"] + assert resolved.response == row["response"] + if cold_has_audit: + assert resolved.proxy_server_request["classifier_input"] == full_audit["classifier_input"] + assert resolved.proxy_server_request["originating_request_masked"] == full_audit["originating_request_masked"] + else: + assert resolved.proxy_server_request == row["proxy_server_request"] + + @pytest.mark.parametrize( "value, expected", [ diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 5a79560b972..adc15870968 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -67,6 +67,30 @@ def _get_additional_usage_values_for_usage(usage: litellm.Usage) -> dict: return metadata["additional_usage_values"] +@pytest.mark.parametrize("store_prompts,redact", [(True, False), (False, False), (True, True)]) +def test_classifier_audit_spend_storage_obeys_privacy_and_truncation(monkeypatch, store_prompts, redact): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": store_prompts}) + audit: Final = { + "classifier_input": {"system": "rubric" * 1000, "messages": [{"role": "user", "content": "ask"}]}, + "originating_request_masked": {"input": "source-only", "api_key": "REDACTED"}, + } + stored: Final = json.loads(_get_proxy_server_request_for_spend_logs_payload( + metadata={}, litellm_params={"proxy_server_request": {"body": {"model": "classifier"}}}, + kwargs={"standard_logging_object": audit, "standard_callback_dynamic_params": {"turn_off_message_logging": redact}}, + )) + if not store_prompts or redact: + assert "classifier_input" not in stored + assert "originating_request_masked" not in stored + else: + assert stored["classifier_input"]["messages"] == audit["classifier_input"]["messages"] + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in json.dumps(stored["classifier_input"]) + assert stored["originating_request_masked"]["input"] == "source-only" + assert stored["model"] == "classifier" + assert audit["classifier_input"]["system"] == "rubric" * 1000 + + def test_get_logging_payload_maps_openai_cached_tokens_to_cache_read_input_tokens(): additional_usage_values = _get_additional_usage_values_for_usage( litellm.Usage( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 51103297c58..45b2faea126 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2940,6 +2940,29 @@ class TestLLMClassifier: "REASONING", ] + @pytest.mark.asyncio + @pytest.mark.parametrize("source_body", [ + {"model": "router", "messages": [{"role": "user", "content": "source-only"}]}, + {"model": "router", "system": "source-only", "messages": [{"role": "user", "content": "ask"}]}, + {"model": "router", "instructions": "source-only", "input": "ask"}, + ]) + async def test_classifier_source_is_masked_and_separate_from_provider_input( + self, llm_complexity_router, mock_router_instance, source_body + ): + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}')) + outcome = await llm_complexity_router.aclassify( + "classify-this-ask", request_kwargs={"proxy_server_request": { + "body": {**source_body, "metadata": {"authorization": "source-secret"}} + }} + ) + assert outcome.cause == "llm_classifier" + call_kwargs = mock_router_instance.acompletion.call_args.kwargs + source = call_kwargs["proxy_server_request"]["originating_request_masked"] + assert source == {**source_body, "metadata": {"authorization": "REDACTED"}} + assert "source-only" not in str(call_kwargs["messages"]) + assert "source-only" not in str(call_kwargs["proxy_server_request"]["body"]) + assert "classify-this-ask" in str(call_kwargs["messages"]) + @pytest.mark.asyncio @pytest.mark.parametrize("reasoning_effort", [None, "none", "low"], ids=["omitted", "none", "low"]) async def test_classifier_reasoning_effort_reaches_only_classifier_call( diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.test.tsx new file mode 100644 index 00000000000..86e98de5557 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.test.tsx @@ -0,0 +1,56 @@ +import { render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import { ClassifierAuditView } from "./ClassifierAuditView"; + +vi.mock("./JsonViewer", () => ({ + JsonViewer: ({ data }: { data: unknown }) => 
{JSON.stringify(data)}
, +})); + +describe("ClassifierAuditView", () => { + it("separates and copies the provider input, source request, and returned verdict", async () => { + const user = userEvent.setup(); + const input = { system: "classification rubric", messages: [{ role: "user", content: "classify this" }] }; + render( + , + ); + const classifier = within(screen.getByRole("region", { name: "Classifier input" })); + expect(classifier.getByText(/classification rubric/)).toBeInTheDocument(); + expect(classifier.queryByText(/source-only/)).not.toBeInTheDocument(); + expect( + within(screen.getByRole("region", { name: "Originating request, credentials masked" })).getByText(/source-only/), + ).toBeInTheDocument(); + expect( + within(screen.getByRole("region", { name: "Classifier response" })).getByText(/a greeting/), + ).toBeInTheDocument(); + await user.click(classifier.getByRole("button", { name: "Copy Classifier input" })); + expect(await navigator.clipboard.readText()).toBe(JSON.stringify(input, null, 2)); + }); + + it("does not present legacy source messages as captured classifier input", () => { + render(); + expect(screen.getAllByText("Not captured or message logging disabled")).toHaveLength(3); + expect(screen.queryByRole("button", { name: "Copy Classifier input" })).not.toBeInTheDocument(); + }); + + it("labels truncated input without marking a complete source request as truncated", () => { + render( + , + ); + expect(within(screen.getByRole("region", { name: "Classifier input" })).getByRole("status")).toHaveTextContent( + "This stored copy is truncated", + ); + expect( + within(screen.getByRole("region", { name: "Originating request, credentials masked" })).queryByRole("status"), + ).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.tsx new file mode 100644 index 00000000000..913700efa9b --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.tsx @@ -0,0 +1,52 @@ +import CopyButton from "@/components/shared/CopyButton"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import type { ReactNode } from "react"; +import { JsonViewer } from "./JsonViewer"; + +interface ClassifierAuditViewProps { + request: Record; + response: unknown; +} + +export function ClassifierAuditView({ request, response }: ClassifierAuditViewProps) { + return ( +
+ + Provider request payload. A cached call or disabled message logging may have no capture. + + + Comparison only. This source request was not appended to the classifier input. + + + The returned verdict and any explanation supplied by the classifier. Later routing rules may change the tier. + +
+ ); +} + +function AuditField({ title, value, children }: { title: string; value: unknown; children: ReactNode }) { + const serialized = JSON.stringify(value); + const truncated = serialized?.includes("litellm_truncated") ?? false; + + return ( + + + {title} + {value != null && } + + +

{children}

+ {truncated && ( +

+ This stored copy is truncated. The complete payload is unavailable from the configured log storage. +

+ )} + {value == null ? ( +

Not captured or message logging disabled

+ ) : ( + + )} + + + ); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index 186cb66592b..e29f2810926 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -48,6 +48,8 @@ import { } from "./constants"; import { ToolsSection } from "../ToolsSection"; import { PrettyMessagesView } from "./PrettyMessagesView"; +import { ClassifierAuditView } from "./ClassifierAuditView"; +import { AUTOROUTER_CLASSIFIER_ORIGIN } from "./ClassifyTag"; export interface LogDetailContentProps { logEntry: LogEntry; @@ -68,6 +70,9 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok const metadata = logEntry.metadata || {}; const hasError = metadata.status === "failure"; const errorInfo = hasError ? metadata.error_information : null; + const isClassifier = + metadata.internal_call_origin === AUTOROUTER_CLASSIFIER_ORIGIN && + (logEntry.call_type === "completion" || logEntry.call_type === "acompletion"); const hasMessages = checkHasMessages(logEntry.messages); const hasResponse = checkHasResponse(logEntry.response); @@ -196,7 +201,11 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok Loading request & response data... - ) : ( + ) : null} + {!isLoadingDetails && isClassifier && ( + + )} + {!isLoadingDetails && !isClassifier && ( - {metadata?.attempted_retries !== undefined && metadata?.attempted_retries !== null ? ( - metadata.attempted_retries > 0 ? ( - <> - {metadata.attempted_retries} - {metadata.max_retries !== undefined && metadata.max_retries !== null - ? ` / ${metadata.max_retries}` - : ""} - - ) : ( - - None - - ) - ) : ( - "-" + {metadata?.attempted_retries != null && metadata.attempted_retries > 0 && ( + <> + {metadata.attempted_retries} + {metadata.max_retries !== undefined && metadata.max_retries !== null + ? ` / ${metadata.max_retries}` + : ""} + )} + {metadata?.attempted_retries != null && metadata.attempted_retries <= 0 && ( + + None + + )} + {metadata?.attempted_retries == null && "-"} @@ -602,16 +609,10 @@ function RequestResponseSection({ const totalTokens = promptTokens + completionTokens; const costBreakdown = logEntry.metadata?.cost_breakdown; const useCostBreakdown = costBreakdown?.input_cost !== undefined && costBreakdown?.output_cost !== undefined; - const inputCost = useCostBreakdown - ? costBreakdown!.input_cost ?? 0 - : totalTokens > 0 - ? (totalSpend * promptTokens) / totalTokens - : 0; - const outputCost = useCostBreakdown - ? costBreakdown!.output_cost ?? 0 - : totalTokens > 0 - ? (totalSpend * completionTokens) / totalTokens - : 0; + const estimatedInputCost = totalTokens > 0 ? (totalSpend * promptTokens) / totalTokens : 0; + const estimatedOutputCost = totalTokens > 0 ? (totalSpend * completionTokens) / totalTokens : 0; + const inputCost = useCostBreakdown ? costBreakdown!.input_cost ?? 0 : estimatedInputCost; + const outputCost = useCostBreakdown ? costBreakdown!.output_cost ?? 0 : estimatedOutputCost; return (
From 1697684b685278ac68764a2f7278ffee9f05d2ff Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 12:01:04 -0700 Subject: [PATCH 047/126] fix(router): scope Codex envelope defaults to Codex clients --- litellm/litellm_core_utils/core_helpers.py | 8 ++ litellm/proxy/litellm_pre_call_utils.py | 15 +-- .../complexity_router/README.md | 4 +- .../complexity_router/complexity_router.py | 40 +++++--- .../complexity_router/config.py | 5 +- .../router_strategy/test_complexity_router.py | 96 +++++++++++++++++-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 7 files changed, 127 insertions(+), 43 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index eacc3e4860a..aa7d6ca1699 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -2,6 +2,7 @@ ## Helper utilities import copy import logging +import re from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, Final, Literal @@ -21,6 +22,13 @@ else: Span = Any +_CODEX_CLIENT_PREFIX_RE: Final = re.compile(r"^codex[-_ /]", re.IGNORECASE) + + +def is_codex_user_agent(user_agent: str) -> bool: + return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent)) + + def safe_divide_seconds(seconds: float, denominator: float, default: float | None = None) -> float | None: """ Safely divide seconds by denominator, handling zero division. diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index da033dc2276..ad4687e95db 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -31,6 +31,7 @@ from litellm.constants import ( SESSION_ID_OMITTED_METADATA_KEY, X_LITELLM_DISABLE_CALLBACKS, ) +from litellm.litellm_core_utils.core_helpers import is_codex_user_agent from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( TRUSTED_CALLBACK_VARS_FIELD, @@ -83,10 +84,6 @@ _EXPLICIT_SESSION_HEADERS: Final = frozenset({"x-litellm-trace-id", "x-litellm-s # ``session-id``/``thread-id``; builds before the codex-api split sent # ``session_id``/``conversation_id``. Ordered session before thread. _CODEX_SESSION_ID_HEADERS: Final = ("session-id", "session_id", "thread-id", "conversation_id") -# Matches every first-party Codex originator: codex-tui, codex_cli_rs, codex_exec, -# codex_vscode, "Codex ...". A separator is required so an unrelated "codexfoo" client -# does not read as Codex. -_CODEX_CLIENT_PREFIX_RE: Final = re.compile(r"^codex[-_ /]", re.IGNORECASE) # Session-id values must be non-empty strings of alphanumerics, hyphens, or underscores # (covers UUIDs and most common session-id formats). _SESSION_ID_VALUE_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") @@ -810,16 +807,6 @@ def apply_missing_session_id_policy( ) -def is_codex_user_agent(user_agent: str) -> bool: - """Codex builds its user agent as ``/ ...`` and ships - several first-party originators: ``codex-tui``, ``codex_cli_rs``, - ``codex_exec`` (exec mode), ``codex_vscode`` (IDE extension) and ``Codex ...`` - (see ``is_first_party_originator`` in codex-rs). They agree only on the - ``codex`` stem, and the TUI sends a bare ``codex-tui`` with no version at all, - so match the stem plus a separator rather than any one spelling.""" - return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent)) - - def should_auto_drop_params_for_agentic_cli(user_agent: str, data: dict, proxy_config: ProxyConfig) -> bool: """drop_params defaults to on for agentic CLIs so their client-specific params (e.g. Claude Code's thinking, Codex's service_tier) don't fail diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 3d2b33a2ecd..be501db04f7 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -446,7 +446,9 @@ Reasoning markers in the system prompt do **not** trigger the reasoning override Agent harnesses inject their own context into the conversation as ordinary message text. That text is plumbing, not something a human asked for, so the router strips complete reminder blocks before classifying and picking a tier. A turn that is nothing but a reminder block strips to empty and is skipped, and the router falls back to the last real ask instead -By default the router strips complete ``, ``, ``, ``, and `` blocks. It also strips Codex repository instructions from the fixed heading prefix `# AGENTS.md instructions for ` through ``, regardless of the repository path +By default the router strips complete `` blocks. For requests with a Codex user agent, it also strips complete ``, ``, ``, and `` blocks, plus repository instructions from the fixed heading prefix `# AGENTS.md instructions for ` through ``, regardless of the repository path. Other clients keep those tags and their contents + +The proxy records the incoming user agent in request metadata. SDK callers can supply `metadata.user_agent` (or `litellm_metadata.user_agent` on Responses requests), or configure `reminder_markers` explicitly when their client identity is unavailable The Codex `Message Type: NEW_TASK` wrapper and its delegated-task payload remain available for classification. Cleanup applies to the current ask and quoted prior turns; the routed request retains its original content diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 923aac85ee2..4c63452c1e9 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -39,6 +39,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, get_metadata_variable_name_from_kwargs, + is_codex_user_agent, ) from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -386,8 +387,8 @@ def _effective_turn_off_message_logging(request_kwargs: Mapping[str, object] | N _REMINDER_OPEN: Final = "" _REMINDER_CLOSE: Final = "" -_DEFAULT_REMINDER_MARKERS: Final = ( - (_REMINDER_OPEN, _REMINDER_CLOSE), +_DEFAULT_REMINDER_MARKERS: Final = ((_REMINDER_OPEN, _REMINDER_CLOSE),) +_CODEX_REMINDER_MARKERS: Final = _DEFAULT_REMINDER_MARKERS + ( ("", ""), ("", ""), ("", ""), @@ -1951,6 +1952,7 @@ class ComplexityRouter(CustomLogger): raise ValueError("classifier_llm_config is not set") include_assistant: Final = self.config.classifier_context_include_assistant_turns + marker_pairs: Final = self._reminder_markers_for_request(request_kwargs or {}) context_enabled: Final = bool(messages) and self.config.classifier_context_window_size > 0 prior_turns: Final = ( _extract_prior_turns( @@ -1960,20 +1962,14 @@ class ComplexityRouter(CustomLogger): budget_chars=self.config.classifier_context_budget_chars, per_turn_chars=self.config.classifier_context_per_turn_chars, include_assistant=include_assistant, - marker_pairs=self._reminder_markers, + marker_pairs=marker_pairs, ) if context_enabled else () ) has_prior_conversation: Final = ( context_enabled - and len( - tuple( - islice( - _iter_context_turns_newest_first(messages or (), include_assistant, self._reminder_markers), 2 - ) - ) - ) + and len(tuple(islice(_iter_context_turns_newest_first(messages or (), include_assistant, marker_pairs), 2))) > 1 ) @@ -2434,7 +2430,7 @@ class ComplexityRouter(CustomLogger): body if isinstance(body, Mapping) else None, resolved_messages, tuple(self.config.plan_mode_patterns or ()), - self._reminder_markers, + self._reminder_markers_for_request(request_kwargs), ) def _matched_housekeeping_sentinel(self, newest_ask: str | None) -> str | None: @@ -3255,6 +3251,18 @@ class ComplexityRouter(CustomLogger): """ return _extract_current_ask_and_system_prompt(messages) + def _reminder_markers_for_request(self, request_kwargs: Mapping[str, object]) -> tuple[tuple[str, str], ...]: + if self.config.reminder_markers is not None: + return self._reminder_markers + if any( + is_codex_user_agent(user_agent) + for metadata_key in ("litellm_metadata", "metadata") + if isinstance(metadata := request_kwargs.get(metadata_key), Mapping) + if isinstance(user_agent := metadata.get("user_agent"), str) + ): + return _CODEX_REMINDER_MARKERS + return _DEFAULT_REMINDER_MARKERS + @staticmethod def _iter_metadata_dicts(request_kwargs: dict) -> list[dict]: """Metadata may land on `metadata` or `litellm_metadata` depending on the @@ -3356,6 +3364,7 @@ class ComplexityRouter(CustomLogger): # chat-completions messages, so it is real work on every non-chat surface, and # both the conversation shape and the classifier read the same list. resolved_messages: Final = self._resolve_messages(messages, request_kwargs) + marker_pairs: Final = self._reminder_markers_for_request(request_kwargs) conversation_continuing: Final = _conversation_is_continuing(resolved_messages) use_session_affinity: Final = self._uses_tier_pin @@ -3365,7 +3374,7 @@ class ComplexityRouter(CustomLogger): # In 'user_turn' mode a held pin is replayed only on continuation turns; a new human # ask falls through and re-classifies. session_affinity restores pin-first for asks too. pin_replay_allowed: Final = bool(self.config.session_affinity) or not _newest_turn_is_human_ask( - resolved_messages, self._reminder_markers + resolved_messages, marker_pairs ) if cache_key is not None and pin_replay_allowed: @@ -3376,7 +3385,7 @@ class ComplexityRouter(CustomLogger): pin_escalation_keyword: str | None = None if self.escalation_keywords: user_message: Final = ( - _newest_turn_ask(resolved_messages, self._reminder_markers) if resolved_messages else None + _newest_turn_ask(resolved_messages, marker_pairs) if resolved_messages else None ) if user_message is not None: pin_escalation_keyword = self._matched_escalation_keyword(user_message) @@ -3564,7 +3573,8 @@ class ComplexityRouter(CustomLogger): # Determine whether the original request used messages directly has_original_messages: Final = messages is not None and len(messages) > 0 - user_message, system_prompt = _extract_current_ask_and_system_prompt(resolved_messages, self._reminder_markers) + marker_pairs: Final = self._reminder_markers_for_request(request_kwargs) + user_message, system_prompt = _extract_current_ask_and_system_prompt(resolved_messages, marker_pairs) classifier_images: Final = self._classifier_image_parts(resolved_messages) if user_message is None and not classifier_images: @@ -3598,7 +3608,7 @@ class ComplexityRouter(CustomLogger): ) ask: Final = user_message or "" - newest_ask: Final = _newest_turn_ask(resolved_messages, self._reminder_markers) + newest_ask: Final = _newest_turn_ask(resolved_messages, marker_pairs) escalation_keyword: Final = self._matched_escalation_keyword(newest_ask) if newest_ask is not None else None # Resolved here rather than beside the classifier because the keyword-override path below # returns before any classification runs, and a forced tier gets stuck for the same reason diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 6ab4aa85148..fb7003c887c 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -1292,8 +1292,9 @@ class ComplexityRouterConfig(BaseModel): "Override the delimiter pairs used to recognize and strip harness-injected reminder " "blocks before classification. A harness that wraps injected context differently per " "agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than " - "adds to, the built-in system-reminder and Codex envelope pairs, so list every " - "built-in pair your harness also emits. Matching is case-insensitive." + "adds to, the built-in system-reminder pair and the Codex envelope pairs enabled " + "for Codex user agents, so list every built-in pair your harness also emits. " + "Matching is case-insensitive." ), ) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 792963fde23..44d7454e836 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -7625,9 +7625,38 @@ _CODEX_ENVELOPES: Final = ( class TestContextAwareClassifier: """Test the new classifier context window and trajectory signals.""" + @pytest.mark.asyncio + @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) + @pytest.mark.parametrize("user_agent", (None, "curl/8.7.1", "codexify/1.0")) + async def test_non_codex_requests_preserve_tagged_asks(self, envelope: str, user_agent: str | None) -> None: + completion: Final = AsyncMock(return_value=_llm_response('{"tier":"COMPLEX"}')) + router: Final = ComplexityRouter( + model_name="router", + litellm_router_instance=MagicMock(acompletion=completion), + complexity_router_config={ + "tiers": {"COMPLEX": "task-model"}, + "default_model": "fallback-model", + "classifier_type": "llm", + "classifier_llm_config": {"model": "classifier-model"}, + "escalation_keywords": [], + }, + ) + + response: Final = await router.async_pre_routing_hook( + model="router", + request_kwargs={"metadata": {"user_agent": user_agent}} if user_agent is not None else {}, + messages=[{"role": "user", "content": envelope}], + ) + + assert response is not None + assert response.model == "task-model" + completion.assert_awaited_once() + assert completion.call_args.kwargs["messages"][1]["content"].strip() == f"Classify this message:\n{envelope}" + @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) def test_codex_envelopes_preserve_delegated_task_and_prior_context(self, envelope: str) -> None: from litellm.router_strategy.complexity_router.complexity_router import ( + _CODEX_REMINDER_MARKERS, _extract_current_ask_and_system_prompt, _extract_prior_turns, _newest_turn_ask, @@ -7644,21 +7673,25 @@ class TestContextAwareClassifier: {"role": "user", "content": envelope}, ] - assert _extract_current_ask_and_system_prompt(messages)[0] == _CODEX_NEW_TASK - assert _extract_prior_turns(messages, _CODEX_NEW_TASK, 1, 100, None, False) == ( + assert _extract_current_ask_and_system_prompt(messages, _CODEX_REMINDER_MARKERS)[0] == _CODEX_NEW_TASK + assert _extract_prior_turns(messages, _CODEX_NEW_TASK, 1, 100, None, False, _CODEX_REMINDER_MARKERS) == ( ("user", "Design cache invalidation"), ) - assert _newest_turn_ask(messages) is None - assert _newest_turn_is_human_ask(messages) is False - assert _extract_current_ask_and_system_prompt([messages[-1]])[0] is None + assert _newest_turn_ask(messages, _CODEX_REMINDER_MARKERS) is None + assert _newest_turn_is_human_ask(messages, _CODEX_REMINDER_MARKERS) is False + assert _extract_current_ask_and_system_prompt([messages[-1]], _CODEX_REMINDER_MARKERS)[0] is None @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) def test_codex_marker_override_and_incomplete_blocks_preserve_text(self, envelope: str) -> None: - from litellm.router_strategy.complexity_router.complexity_router import _strip_reminder_blocks + from litellm.router_strategy.complexity_router.complexity_router import ( + _CODEX_REMINDER_MARKERS, + _strip_reminder_blocks, + ) incomplete: Final = envelope.rsplit("noise{envelope}", (("", ""),)) == envelope @pytest.mark.asyncio @@ -7681,9 +7714,12 @@ class TestContextAwareClassifier: ] original: Final = deepcopy(messages) request_kwargs: Final = ( - {"input": messages, "litellm_metadata": {"user_api_key_request_route": "/v1/responses"}} + { + "input": messages, + "litellm_metadata": {"user_api_key_request_route": "/v1/responses", "user_agent": "codex-tui"}, + } if responses_api - else {} + else {"metadata": {"user_agent": "codex-tui"}} ) response: Final = await router.async_pre_routing_hook( @@ -7706,6 +7742,46 @@ class TestContextAwareClassifier: else: assert response.messages == original + @pytest.mark.asyncio + @pytest.mark.parametrize("custom_markers", (False, True)) + async def test_codex_markers_are_request_scoped_and_respect_overrides(self, custom_markers: bool) -> None: + completion: Final = AsyncMock(return_value=_llm_response('{"tier":"COMPLEX"}')) + router: Final = ComplexityRouter( + model_name="router", + litellm_router_instance=MagicMock(acompletion=completion), + complexity_router_config={ + "tiers": {"COMPLEX": "task-model"}, + "classifier_type": "llm", + "classifier_llm_config": {"model": "classifier-model"}, + "classifier_context_window_size": 2, + "escalation_keywords": [], + **({"reminder_markers": [{"open": "", "close": ""}]} if custom_markers else {}), + }, + ) + envelope: Final = "\n".join(_CODEX_ENVELOPES) + prior: Final = f"{envelope}\nDesign cache invalidation" + messages: Final = [ + {"role": "user", "content": prior}, + {"role": "user", "content": _CODEX_NEW_TASK}, + {"role": "user", "content": envelope}, + ] + for user_agent in ("codex-tui", "curl/8.7.1", "codex_cli_rs/0.62.0"): + response: Final = await router.async_pre_routing_hook( + model="router", request_kwargs={"metadata": {"user_agent": user_agent}}, messages=messages + ) + assert response is not None + assert response.model == "task-model" + payload: Final = completion.call_args.kwargs["messages"][1]["content"] + if user_agent.startswith("codex") and not custom_markers: + assert payload.endswith(f"Classify this message:\n{_CODEX_NEW_TASK}") + assert "Design cache invalidation" in payload + assert "LITELLM ESCALATE" not in payload + else: + assert payload.endswith(f"Classify this message:\n{envelope}") + assert prior in payload + assert response.messages == messages + assert completion.await_count == 3 + @pytest.mark.parametrize( "messages,expected_ask", [ diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 5c4661ab357..ab2c1f27a59 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35221,7 +35221,7 @@ export interface components { reasoning_override_min_score?: number | null; /** * Reminder Markers - * @description Override the delimiter pairs used to recognize and strip harness-injected reminder blocks before classification. A harness that wraps injected context differently per agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than adds to, the built-in default of ('', ''), so a harness that also emits that pair lists it too. Matching is case-insensitive. + * @description Override the delimiter pairs used to recognize and strip harness-injected reminder blocks before classification. A harness that wraps injected context differently per agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than adds to, the built-in system-reminder pair and the Codex envelope pairs enabled for Codex user agents, so list every built-in pair your harness also emits. Matching is case-insensitive. */ reminder_markers?: components["schemas"]["ReminderMarkerPair"][] | null; /** From 41a147f2a0abfe1455e9560252c090d175d650d3 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 12:22:17 -0700 Subject: [PATCH 048/126] fix(router): normalize Azure classifier audit payloads --- litellm/llms/azure/azure.py | 2 ++ .../test_litellm_logging.py | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 46a9dd1a531..587165e6991 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -323,6 +323,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): "api_version": api_version, "api_base": api_base, "complete_input_dict": data, + "openai_sdk": True, }, ) if not isinstance(max_retries, int): @@ -429,6 +430,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): "api_base": api_base, "acompletion": True, "complete_input_dict": data, + "openai_sdk": True, }, ) diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 28545bb2349..6de0fd3d4e0 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -6804,11 +6804,11 @@ def test_get_error_information_redacts_provider_key_from_upstream_url(): @pytest.mark.asyncio -@pytest.mark.parametrize("provider", ["openai", "anthropic", "bedrock"]) +@pytest.mark.parametrize("provider", ["openai", "azure", "anthropic", "bedrock"]) async def test_classifier_audit_matches_provider_transport(provider: str) -> None: import json - from openai import AsyncOpenAI + from openai import AsyncAzureOpenAI, AsyncOpenAI from litellm.litellm_core_utils.classifier_logging import classifier_input_snapshot from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -6844,9 +6844,17 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non handler: Final = AsyncHTTPHandler() await handler.close() handler.client = http_client - client: Final = AsyncOpenAI(api_key="transport-only", http_client=http_client) if provider == "openai" else handler + client: Final = ( + AsyncAzureOpenAI( + api_key="transport-only", azure_endpoint="https://azure.invalid", + api_version="2025-04-01-preview", http_client=http_client, + ) + if provider == "azure" else AsyncOpenAI(api_key="transport-only", http_client=http_client) + if provider == "openai" else handler + ) model: Final = { "openai": "openai/gpt-5.6", + "azure": "azure/gpt-5.6", "anthropic": "anthropic/claude-haiku-4-5", "bedrock": "bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", }[provider] @@ -6859,8 +6867,9 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non metadata={"internal_call_origin": "autorouter_classifier"}, proxy_server_request={"body": {}, "originating_request_masked": {"input": f"source-only-{marker}"}}, success_callback=[capture], num_retries=0, + **({"api_base": "https://azure.invalid", "api_version": "2025-04-01-preview"} if provider == "azure" else {}), **({"extra_body": {"audit_context": "provider-extra"}, "extra_headers": {"X-Audit": "header-only-secret"}} - if provider == "openai" else {}), + if provider in ("openai", "azure") else {}), ) await asyncio.gather(run("request-one"), run("request-two")) @@ -6877,7 +6886,7 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non marker: Final = "request-one" if "request-one" in json.dumps(snapshot) else "request-two" assert payload["originating_request_masked"] == {"input": f"source-only-{marker}"} assert classifier_input_snapshot(snapshot) is not None - if provider != "openai": + if provider not in ("openai", "azure"): assert all("system" in request for request in requests) From 181b3fd94a17e54f0acbf42776bef36b5da4f2ed Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 12:24:42 -0700 Subject: [PATCH 049/126] fix(router): classify new asks before reminder-only tails --- .../complexity_router/README.md | 2 + .../complexity_router/complexity_router.py | 32 +++++-- .../router_strategy/test_complexity_router.py | 96 +++++++++++++++++++ 3 files changed, 121 insertions(+), 9 deletions(-) diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index be501db04f7..f92e3569bbe 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -452,6 +452,8 @@ The proxy records the incoming user agent in request metadata. SDK callers can s The Codex `Message Type: NEW_TASK` wrapper and its delegated-task payload remain available for classification. Cleanup applies to the current ask and quoted prior turns; the routed request retains its original content +In `classification_mode: user_turn`, complete text-only reminder tails leave the preceding fresh ask eligible for classification. Assistant turns and tool results still mark continuations, including tool results carried alongside reminder text + `reminder_markers` replaces these defaults with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit: ```yaml diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 4c63452c1e9..a03b01ff1e0 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -599,6 +599,18 @@ def _last_human_ask_index( ) +def _is_reminder_only_turn(message: Mapping[str, object], marker_pairs: tuple[tuple[str, str], ...]) -> bool: + if message.get("role") != "user": + return False + content: Final = message.get("content") + if not isinstance(content, str) and not ( + isinstance(content, list) and all(isinstance(part, Mapping) and part.get("type") == "text" for part in content) + ): + return False + text: Final = _message_text(content) + return bool(text.strip()) and not _strip_reminder_blocks(text, marker_pairs) + + def _newest_turn_is_human_ask( messages: Sequence[Mapping[str, object]] | None, marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, @@ -609,21 +621,23 @@ def _newest_turn_is_human_ask( Anchored on `_last_human_ask_index` so every surface's plumbing reads as a continuation: chat-completions tool turns are role=tool, Messages-surface tool_result turns flatten to empty human text, and a hybrid turn carrying an ask alongside a tool_result still counts as an ask. - Compared against the newest non-system message rather than the raw tail, because Claude Code - appends a system-role reminder after the human turn; that trailing plumbing is neither an ask - nor loop traffic and must not turn a fresh ask into a continuation. An unreadable request (no - messages) is treated as a continuation: there is no ask to classify, which is the same reading - `_extract_current_ask_and_system_prompt` gives it downstream. + Trailing system messages and complete text-only reminders do not turn a fresh ask into a + continuation. Assistant turns and non-text content, including tool results alongside reminders, + still form continuation boundaries. An unreadable request has no ask to classify. """ if not messages: return False - newest_non_system: Final = next( - (index for index in range(len(messages) - 1, -1, -1) if messages[index].get("role") != "system"), + newest_activity: Final = next( + ( + index + for index in range(len(messages) - 1, -1, -1) + if messages[index].get("role") != "system" and not _is_reminder_only_turn(messages[index], marker_pairs) + ), None, ) - if newest_non_system is None: + if newest_activity is None: return False - return _last_human_ask_index(messages, marker_pairs) == newest_non_system + return _last_human_ask_index(messages, marker_pairs) == newest_activity def _iter_system_scope_texts( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 44d7454e836..750b4780c04 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -7625,6 +7625,102 @@ _CODEX_ENVELOPES: Final = ( class TestContextAwareClassifier: """Test the new classifier context window and trajectory signals.""" + @pytest.mark.parametrize( + "tail,expected", + ( + ([{"role": "user", "content": [{"type": "text", "text": _CODEX_ENVELOPES[0]}]}], True), + ([{"role": "assistant", "content": _CODEX_ENVELOPES[0]}], False), + ([{"role": "tool", "content": _CODEX_ENVELOPES[0]}], False), + ([{"role": "user", "content": " "}], False), + ( + [{"role": "user", "content": [_TOOL_RESULT, {"type": "text", "text": _CODEX_ENVELOPES[0]}]}], + False, + ), + ( + [{"role": "user", "content": [{"type": "image_url"}, {"type": "text", "text": _CODEX_ENVELOPES[0]}]}], + False, + ), + ), + ) + def test_only_text_reminder_tails_are_ignored_for_new_asks(self, tail: list[dict[str, object]], expected: bool) -> None: + from litellm.router_strategy.complexity_router.complexity_router import ( + _CODEX_REMINDER_MARKERS, + _newest_turn_is_human_ask, + ) + + assert _newest_turn_is_human_ask([_ASKED, *tail], _CODEX_REMINDER_MARKERS) is expected + assert _newest_turn_is_human_ask(tail, _CODEX_REMINDER_MARKERS) is False + + @pytest.mark.asyncio + @pytest.mark.parametrize("new_ask", (_CODEX_NEW_TASK, "Now design cache invalidation")) + @pytest.mark.parametrize("responses_api", (False, True)) + @pytest.mark.parametrize("session_affinity", (False, True)) + async def test_codex_tail_preserves_new_ask_and_tool_continuation_boundaries( + self, new_ask: str, responses_api: bool, session_affinity: bool + ) -> None: + completion: Final = AsyncMock( + side_effect=[_llm_response('{"tier":"SIMPLE"}'), _llm_response('{"tier":"COMPLEX"}')] + ) + router: Final = ComplexityRouter( + model_name="router", + litellm_router_instance=MagicMock(acompletion=completion, cache=DualCache()), + complexity_router_config={ + "tiers": {"SIMPLE": "simple-model", "COMPLEX": "task-model"}, + "classifier_type": "llm", + "classifier_llm_config": {"model": "classifier-model"}, + "classification_mode": "user_turn", + "session_affinity": session_affinity, + "escalation_keywords": [], + }, + ) + metadata: Final = {"user_agent": "codex-tui", "session_id": "codex-tail-session"} + first_messages: Final = [{"role": "user", "content": "Hello"}] + tail: Final = [{"role": "user", "content": envelope} for envelope in _CODEX_ENVELOPES] + new_messages: Final = [ + *first_messages, + {"role": "assistant", "content": "Hello"}, + {"role": "user", "content": new_ask}, + *tail, + ] + continuation: Final = [ + *new_messages, + {"role": "assistant", "content": "Working on it"}, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "read-cache", "content": "cache source"}, + {"type": "text", "text": _CODEX_ENVELOPES[0]}, + ], + }, + *tail, + ] + results: Final = [ + await router.async_pre_routing_hook( + model="router", + request_kwargs=( + {"input": messages, "litellm_metadata": {**metadata, "user_api_key_request_route": "/v1/responses"}} + if responses_api + else {"metadata": metadata} + ), + messages=None if responses_api else messages, + input=messages if responses_api else None, + ) + for messages in (first_messages, new_messages, continuation) + ] + + assert [result.model for result in results] == ( + ["simple-model", "simple-model", "simple-model"] + if session_affinity + else ["simple-model", "task-model", "task-model"] + ) + assert completion.await_count == (1 if session_affinity else 2) + assert results[-1].routing_decision["cause"] == ( + "session_affinity_pin" if session_affinity else "user_turn_continuation" + ) + if not session_affinity: + assert completion.call_args.kwargs["messages"][1]["content"].endswith(f"Classify this message:\n{new_ask}") + assert results[1].messages == (None if responses_api else new_messages) + @pytest.mark.asyncio @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) @pytest.mark.parametrize("user_agent", (None, "curl/8.7.1", "codexify/1.0")) From f4ebcef0a17c2952ab90100fdba702d863c728f9 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 12:32:09 -0700 Subject: [PATCH 050/126] fix(router): classify encrypted delegated tasks with native Responses --- .../transformation.py | 3 + .../complexity_router/README.md | 10 + .../complexity_router/complexity_router.py | 119 ++++++++++-- .../router_strategy/test_complexity_router.py | 178 ++++++++++++++++++ 4 files changed, 293 insertions(+), 17 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index c3c18e3d009..5a6debc4af5 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1201,6 +1201,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Cast to Any to match the expected union type for tools list items tools.append(cast(Any, web_search_tool)) + def transform_response_format_to_text_format(self, response_format: object) -> "ResponseText | None": + return self._transform_response_format_to_text_format(response_format) + def _transform_response_format_to_text_format(self, response_format: object) -> "ResponseText | None": """ Transform Chat Completion response_format parameter to Responses API text.format parameter. diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index d605b43e42a..79119008aa2 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -361,6 +361,16 @@ model_list: keep the classifier deployment or provider default, or set a supported value such as `none` or `low` to override that call. +When the current ask is a Responses API `agent_message` containing `encrypted_content`, LLM +classification preserves the encrypted task and uses native Responses. This also bypasses the +local scoring shortcut in `heuristic_first` and `hybrid` modes. The configured classifier must use +a native OpenAI or Azure OpenAI Responses deployment with access to the encrypted content. The +provider handles the encrypted task, and the classifier still chooses the tier dynamically + +Unsupported classifier deployments and provider decryption errors use the existing +`classifier_fallback` policy. No fixed tier is introduced for encrypted tasks. Plaintext asks and +requests carrying only historical encrypted reasoning retain the existing classifier path + Classifier calls have a one-attempt hard deadline. After a timeout, the router opens a process-local circuit for that classifier and sends every session through `classifier_fallback` for `classifier_llm_config.circuit_breaker_cooldown_seconds` (30 seconds by default). When the cooldown diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index faafcea404a..3babe5b96ed 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -25,7 +25,7 @@ from threading import Lock from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast -from pydantic import BaseModel, create_model +from pydantic import BaseModel, TypeAdapter, create_model from litellm._logging import verbose_router_logger from litellm.constants import ( @@ -56,6 +56,7 @@ from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionImageObject, ChatCompletionTextObject, + ResponsesAPIResponse, ) from litellm.types.utils import ( AUTOROUTER_CLASSIFIER_CALL_ORIGIN, @@ -364,7 +365,7 @@ def _parent_session_kwargs(request_kwargs: Mapping[str, Any] | None) -> Mapping[ return {k: kwargs[k] for k in ("litellm_session_id", "litellm_trace_id") if kwargs.get(k) is not None} -def _response_cost_or_none(response: ModelResponse) -> float | None: +def _response_cost_or_none(response: ModelResponse | ResponsesAPIResponse) -> float | None: hidden_params: Final = response._hidden_params if not isinstance(hidden_params, dict): return None @@ -486,6 +487,36 @@ def _human_text(content: object, marker_pairs: tuple[tuple[str, str], ...] = _DE return _strip_reminder_blocks(_message_text(content), marker_pairs) +def _encrypted_classifier_task( + request_kwargs: Mapping[str, object] | None, + marker_pairs: tuple[tuple[str, str], ...], +) -> dict[str, object] | None: + from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages + + raw_input: Final = (request_kwargs or EMPTY_MAPPING).get("input") + if not isinstance(raw_input, list) or (request_kwargs or EMPTY_MAPPING).get("messages"): + return None + items: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(raw_input) + current: Final = next( + ( + item + for item in reversed(items) + if (messages := resolve_structured_messages(messages=None, request_kwargs={"input": [item]})) + and any(_iter_human_asks_newest_first(messages, marker_pairs)) + ), + None, + ) + if current is None or current.get("type") != "agent_message" or not isinstance(current.get("content"), list): + return None + parts: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(current["content"]) + if not any(part.get("type") == "encrypted_content" and part.get("encrypted_content") for part in parts): + return None + return { + **current, + "content": [part for part in parts if part.get("type") in ("input_text", "encrypted_content")], + } + + def _iter_human_asks_newest_first( messages: Sequence[Mapping[str, object]], marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, @@ -1630,6 +1661,10 @@ class ComplexityRouter(CustomLogger): return self._classify_with_heuristic_v2(prompt) if self.config.classifier_type == "custom": return await self._classify_with_plugin(prompt, system_prompt, request_kwargs, raw_messages) + if self.config.classifier_type in ("heuristic_first", "hybrid") and _encrypted_classifier_task( + request_kwargs, self._reminder_markers + ): + return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages) if self.config.classifier_type == "heuristic_first" and self.config.classifier_llm_config is not None: return await self._classify_heuristic_first(prompt, system_prompt, request_kwargs, messages) if self.config.classifier_type == "hybrid" and self.config.classifier_llm_config is not None: @@ -1970,8 +2005,9 @@ class ComplexityRouter(CustomLogger): > 1 ) + encrypted_task: Final = _encrypted_classifier_task(request_kwargs, self._reminder_markers) user_payload: Final = self._build_classifier_user_payload( - prompt=prompt, + prompt="The delegated task in the following agent_message." if encrypted_task is not None else prompt, system_prompt=system_prompt, prior_turns=prior_turns, messages=messages, @@ -2004,34 +2040,37 @@ class ComplexityRouter(CustomLogger): if llm_config.reasoning_effort is not None: classifier_call_params = MappingProxyType({"reasoning_effort": llm_config.reasoning_effort}) - proxy_server_request: Final = { - "body": { - "model": llm_config.model, - "messages": messages_for_call, - "response_format": response_format, - **classifier_call_params, - } - } + payload: Final = ( + self._native_classifier_payload(llm_config.model, messages_for_call, response_format, encrypted_task) + if encrypted_task is not None + else {"messages": messages_for_call, "response_format": response_format, **classifier_call_params} + ) + proxy_server_request: Final = {"body": {"model": llm_config.model, **payload}} + classify: Final = ( + self.litellm_router_instance.aresponses + if encrypted_task is not None + else self.litellm_router_instance.acompletion + ) classifier_timeout_s: Final[float] = llm_config.timeout_ms / 1000 - response: Final[ModelResponse] = await asyncio.wait_for( - self.litellm_router_instance.acompletion( + response: Final[ModelResponse | ResponsesAPIResponse] = await asyncio.wait_for( + classify( model=llm_config.model, - messages=messages_for_call, stream=False, - response_format=response_format, timeout=classifier_timeout_s, num_retries=0, disable_fallbacks=True, metadata=metadata, proxy_server_request=proxy_server_request, turn_off_message_logging=turn_off_message_logging, - **classifier_call_params, + **payload, **_parent_session_kwargs(request_kwargs), ), timeout=classifier_timeout_s, ) - content: Final = response.choices[0].message.content + content: Final = ( + response.output_text if isinstance(response, ResponsesAPIResponse) else response.choices[0].message.content + ) if not content: raise ValueError("LLM classifier returned empty content") raw_tier: Final = _LabeledTierClassification.model_validate_json(content).tier @@ -2040,6 +2079,52 @@ class ComplexityRouter(CustomLogger): raise ValueError(f"LLM classifier returned an unrecognized tier: {raw_tier!r}") return tier, _response_cost_or_none(response) + def _native_classifier_payload( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: existing transformation accepts the SDK message list + response_format: Mapping[str, object], + encrypted_task: Mapping[str, object], + ) -> Mapping[str, object]: + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, get_llm_provider + from litellm.types.router import LiteLLM_Params + + deployments: Final = self._group_deployments(model) + if not deployments: + raise ValueError("Encrypted task classification requires a native OpenAI Responses classifier deployment") + for params in (LiteLLM_Params.model_validate(deployment.get("litellm_params")) for deployment in deployments): + if declared_authenticating_provider(params.model, params.custom_llm_provider): + raise ValueError( + "Encrypted task classification requires a native OpenAI Responses classifier deployment" + ) + _, provider, _, _ = get_llm_provider(model=params.model, litellm_params=params) + if ( + provider not in ("openai", "azure") + or params.use_chat_completions_api + or params.model.startswith("openai/chat_completions/") + ): + raise ValueError( + "Encrypted task classification requires a native OpenAI Responses classifier deployment" + ) + transformation: Final = LiteLLMResponsesTransformationHandler() + input_items, instructions = transformation.convert_chat_completion_messages_to_responses_api(messages) + llm_config: Final = self.config.classifier_llm_config + reasoning: Final = ( + {"reasoning": {"effort": llm_config.reasoning_effort}} + if llm_config is not None and llm_config.reasoning_effort is not None + else {} + ) + return { + "input": [*input_items, encrypted_task], + "instructions": instructions, + "text": transformation.transform_response_format_to_text_format(dict(response_format)), + "store": False, + **reasoning, + } + @staticmethod def _build_classifier_user_payload( prompt: str, diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 51103297c58..87438ef5c69 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -5,6 +5,8 @@ Tests the rule-based complexity scoring and tier assignment logic. """ import asyncio +import copy +import json import logging import sys import time @@ -66,6 +68,7 @@ from litellm.types.router import ( LiteLLM_Params, TaggedPreRoutingStrategy, ) +from litellm.types.llms.openai import ResponsesAPIResponse requires_semantic_router = pytest.mark.skipif( @@ -2482,6 +2485,181 @@ class TestTierLabels: assert set(config.tier_boundaries) == {"simple_medium", "medium_complex", "complex_reasoning"} +def _encrypted_agent_task() -> dict[str, object]: + return { + "type": "agent_message", + "author": "/root", + "recipient": "/root/child", + "content": [ + {"type": "input_text", "text": "Message Type: NEW_TASK\nTask name: /root/child\nPayload:\nHello"}, + {"type": "encrypted_content", "encrypted_content": "opaque-provider-task"}, + ], + } + + +def _native_classifier_response(content: str) -> ResponsesAPIResponse: + response: Final = ResponsesAPIResponse( + id="resp_classifier", + created_at=0, + status="completed", + output=[{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": content}]}], + ) + response._hidden_params = {"response_cost": 0.0001} + return response + + +def _native_classifier_router( + output: str = '{"tier":"REASONING"}', + classifier_type: str = "llm", + deployment_model: str = "openai/gpt-6-astra", + failure: Exception | None = None, +) -> tuple[ComplexityRouter, MagicMock]: + dependency: Final = MagicMock( + aresponses=AsyncMock(return_value=_native_classifier_response(output), side_effect=failure), + acompletion=AsyncMock(return_value=_llm_response('{"tier":"SIMPLE"}')), + get_model_list=MagicMock(return_value=[{"litellm_params": {"model": deployment_model}}]), + ) + return ( + ComplexityRouter( + model_name="encrypted-router", + litellm_router_instance=dependency, + complexity_router_config={ + "tiers": {"SIMPLE": "cheap-model", "REASONING": "deep-model"}, + "classifier_type": classifier_type, + "classifier_llm_config": {"model": "classifier", "timeout_ms": 100, "reasoning_effort": "low"}, + "heuristic_first_max_tier": "SIMPLE" if classifier_type == "heuristic_first" else None, + "hybrid_boundary_margin": 0.01 if classifier_type == "hybrid" else None, + "classifier_fallback": "default_model", + "default_model": "deep-model", + "session_affinity": False, + "deployment_affinity": False, + }, + ), + dependency, + ) + + +class TestEncryptedTaskClassifier: + @pytest.mark.asyncio + @pytest.mark.parametrize("classifier_type", ["llm", "heuristic_first", "hybrid"]) + @pytest.mark.parametrize("tier,model", [("SIMPLE", "cheap-model"), ("REASONING", "deep-model")]) + async def test_encrypted_task_routes_by_native_verdict(self, classifier_type: str, tier: str, model: str): + router, dependency = _native_classifier_router(json.dumps({"tier": tier}), classifier_type) + task: Final = _encrypted_agent_task() + request: Final = { + "input": [ + {"role": "user", "content": "Prior task context"}, + task, + {"type": "function_call_output", "call_id": "call_1", "output": "Tool output"}, + {"role": "user", "content": "Injected reminder"}, + ], + "instructions": "Caller constraints", + "tools": [{"type": "function", "name": "execute"}], + "previous_response_id": "resp_parent", + "litellm_session_id": "parent-session", + "litellm_trace_id": "parent-trace", + "turn_off_message_logging": True, + "litellm_metadata": {"user_api_key_hash": "caller-key-hash"}, + } + original: Final = copy.deepcopy(request) + + result: Final = await router.async_pre_routing_hook(model="encrypted-router", request_kwargs=request) + + assert result.model == model + assert result.routing_decision["tier"] == tier + assert result.routing_decision["cause"] == "llm_classifier" + assert result.routing_decision["classifier_cost"] == 0.0001 + assert result.messages is None + assert request == original + dependency.acompletion.assert_not_called() + call: Final = dependency.aresponses.call_args.kwargs + assert call["input"][-1] == task + assert "opaque-provider-task" not in json.dumps(call["input"][:-1]) + assert "Prior task context" in json.dumps(call["input"][:-1]) + assert "Caller constraints" in json.dumps(call["input"][:-1]) + assert "Caller constraints" not in call["instructions"] + assert "SIMPLE" in call["instructions"] and "REASONING" in call["instructions"] + assert call["text"]["format"]["schema"]["properties"]["tier"]["enum"] == [ + "SIMPLE", "MEDIUM", "COMPLEX", "REASONING" + ] + assert call["text"]["format"]["strict"] is True + assert call["reasoning"] == {"effort": "low"} + assert call["store"] is False + assert call["stream"] is False + assert "tools" not in call and "previous_response_id" not in call + assert "messages" not in call and "response_format" not in call + assert call["timeout"] == 0.1 and call["num_retries"] == 0 and call["disable_fallbacks"] is True + assert call["litellm_session_id"] == "parent-session" + assert call["litellm_trace_id"] == "parent-trace" + assert call["turn_off_message_logging"] is True + assert call["metadata"]["user_api_key_hash"] == "caller-key-hash" + assert call["proxy_server_request"]["body"]["input"] == call["input"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "items", + [ + [{"type": "reasoning", "encrypted_content": "opaque-history", "summary": []}, {"role": "user", "content": "hi"}], + [_encrypted_agent_task(), {"role": "user", "content": "hi"}], + [{**_encrypted_agent_task(), "content": [{"type": "input_text", "text": "hi"}]}], + [{"role": "user", "content": "gAAAA is plain text"}], + [{"role": "user", "content": "hi"}, {"type": "function_call_output", "call_id": "call_1", "output": "opaque-provider-task"}], + ], + ids=["historical-reasoning", "older-encrypted-task", "plaintext-agent", "ciphertext-looking-text", "tool-output"], + ) + async def test_other_asks_keep_chat_classifier(self, items: list[dict[str, object]]): + router, dependency = _native_classifier_router() + + result: Final = await router.async_pre_routing_hook(model="encrypted-router", request_kwargs={"input": items}) + + assert result.model == "cheap-model" + assert result.routing_decision["cause"] == "llm_classifier" + dependency.aresponses.assert_not_called() + dependency.acompletion.assert_awaited_once() + + @pytest.mark.asyncio + @pytest.mark.parametrize("output", ["", "not-json", '{"tier":"UNKNOWN"}']) + async def test_invalid_native_verdict_uses_existing_fallback(self, output: str): + router, dependency = _native_classifier_router(output=output) + + result: Final = await router.async_pre_routing_hook( + model="encrypted-router", request_kwargs={"input": [_encrypted_agent_task()]} + ) + + assert result.model == "deep-model" + assert result.routing_decision["cause"] == "default_model_fallback" + dependency.aresponses.assert_awaited_once() + dependency.acompletion.assert_not_called() + + @pytest.mark.asyncio + @pytest.mark.parametrize("deployment_model", ["anthropic/test-classifier", "openai/chat_completions/gpt-6-astra"]) + async def test_incompatible_classifier_does_not_flatten_encryption(self, deployment_model: str): + router, dependency = _native_classifier_router(deployment_model=deployment_model) + + result: Final = await router.async_pre_routing_hook( + model="encrypted-router", request_kwargs={"input": [_encrypted_agent_task()]} + ) + + assert result.model == "deep-model" + assert result.routing_decision["cause"] == "default_model_fallback" + dependency.aresponses.assert_not_called() + dependency.acompletion.assert_not_called() + + @pytest.mark.asyncio + @pytest.mark.parametrize("failure", [ValueError("invalid_encrypted_content"), TimeoutError("classifier timed out")]) + async def test_native_provider_failure_uses_existing_fallback(self, failure: Exception): + router, dependency = _native_classifier_router(failure=failure) + + result: Final = await router.async_pre_routing_hook( + model="encrypted-router", request_kwargs={"input": [_encrypted_agent_task()]} + ) + + assert result.model == "deep-model" + assert result.routing_decision["cause"] == "default_model_fallback" + dependency.aresponses.assert_awaited_once() + dependency.acompletion.assert_not_called() + + class TestLLMClassifier: """Test the LLM-based classifier path (aclassify) and its fallback behavior.""" From 33954d9cfe3eb887c26ab6a26dcd928ef213638a Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 12:35:59 -0700 Subject: [PATCH 051/126] refactor(router): validate classifier snapshots without recursion --- litellm/litellm_core_utils/classifier_logging.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/classifier_logging.py b/litellm/litellm_core_utils/classifier_logging.py index b9cb52a190d..c0c93161099 100644 --- a/litellm/litellm_core_utils/classifier_logging.py +++ b/litellm/litellm_core_utils/classifier_logging.py @@ -13,13 +13,13 @@ _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) def classifier_input_snapshot(value: object, *, openai_sdk: bool = False) -> Mapping[str, JsonValue] | None: - if openai_sdk and isinstance(value, Mapping): - body: Final = { - key: item for key, item in value.items() if key not in ("extra_headers", "extra_query", "extra_body") - } - extra_body: Final = value.get("extra_body") - return classifier_input_snapshot({**body, **extra_body} if isinstance(extra_body, Mapping) else body) try: + if openai_sdk and isinstance(value, Mapping): + body: Final = { + key: item for key, item in value.items() if key not in ("extra_headers", "extra_query", "extra_body") + } + extra_body: Final = value.get("extra_body") + return _JSON_OBJECT.validate_python({**body, **extra_body} if isinstance(extra_body, Mapping) else body) return ( _JSON_OBJECT.validate_json(value) if isinstance(value, (str, bytes)) From 83b1e9e0d7ac48e4806a91d1e1d7587725a82039 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 12:46:50 -0700 Subject: [PATCH 052/126] test(router): keep classifier E2E verification local --- tests/e2e/coverage_registry/reliability.yaml | 1 - tests/e2e/router/test_classifier_audit_e2e.py | 143 ------------------ 2 files changed, 144 deletions(-) delete mode 100644 tests/e2e/router/test_classifier_audit_e2e.py diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 4d40f19e67b..6b69677d490 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -1,5 +1,4 @@ # Reliability & Performance (behavior features). Grounded in litellm/router.py + router_strategy/ + router_utils/. -- {id: reliability.routing.classifier_audit.separates_provider_input_and_source, module: reliability, tier: P1, behavior: routing, variant: classifier_audit, assertions: [separates_provider_input_and_source], exercised_on: [chat_completions, messages, responses], source: "litellm/litellm_core_utils/classifier_logging.py", rationale: "Classifier spend details distinguish provider input from the credential-masked originating request across all three request surfaces"} - {id: reliability.fallback.5xx.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: "5xx", assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:2024", rationale: "Reroute on provider 5xx to alternate deployment"} - {id: reliability.fallback.context_window.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: context_window, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6108", rationale: "Fallback when model exceeds context limit"} - {id: reliability.fallback.content_policy.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: content_policy, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6023", rationale: "Reroute on content-policy violation"} diff --git a/tests/e2e/router/test_classifier_audit_e2e.py b/tests/e2e/router/test_classifier_audit_e2e.py deleted file mode 100644 index 6b444441477..00000000000 --- a/tests/e2e/router/test_classifier_audit_e2e.py +++ /dev/null @@ -1,143 +0,0 @@ -import json -import os -from collections.abc import Iterator -from contextlib import ExitStack -from dataclasses import dataclass -from typing import Final, Literal - -import pytest -from pydantic import BaseModel, Field, JsonValue, TypeAdapter - -from e2e_config import unique_marker -from e2e_http import AnthropicHeaders, NoBody, unwrap -from models import ChatMessage, KeyGenerateBody, LiteLLMParamsBody -from proxy_client import ProxyClient - -pytestmark = pytest.mark.e2e - - -class AuditMetadata(BaseModel): - source_marker: str - authorization: str = "synthetic-audit-secret" - - -class AuditHeaders(AnthropicHeaders): - enable_redaction: str | None = Field(default=None, serialization_alias="x-litellm-enable-message-redaction") - - -class AuditRequest(BaseModel): - model: str - messages: list[ChatMessage] | None = None - system: str | None = None - instructions: str | None = None - input: str | None = None - max_tokens: int | None = None - max_output_tokens: int | None = None - metadata: AuditMetadata | None = None - litellm_metadata: AuditMetadata | None = None - - -class AuditResponse(BaseModel): - id: str - - -class AuditDetail(BaseModel): - proxy_server_request: dict[str, JsonValue] | str | None = None - response: dict[str, JsonValue] | str | None = None - - -@dataclass(frozen=True, slots=True) -class AuditDeployment: - alias: str - key: str - - -@pytest.fixture -def audit_deployment(proxy: ProxyClient, provider: str) -> Iterator[AuditDeployment]: - marker: Final = unique_marker() - classifier: Final = f"audit-classifier-{marker}" - target: Final = f"audit-target-{marker}" - alias: Final = f"audit-router-{marker}" - model: Final = os.environ.get( - f"E2E_CHEAP_{provider.upper()}_MODEL", "gpt-5.6" if provider == "openai" else "claude-haiku-4-5" - ) - params: Final = LiteLLMParamsBody( - model=f"{provider}/{model}", - api_key=os.environ.get(f"{provider.upper()}_API_KEY") or f"os.environ/{provider.upper()}_API_KEY", - api_base=os.environ.get(f"{provider.upper()}_API_BASE"), - ) - with ExitStack() as stack: - for name in (classifier, target): - stack.callback(proxy.delete_model, proxy.create_model(name, params)) - stack.callback(proxy.delete_model, proxy.create_model(alias, LiteLLMParamsBody( - model="auto_router/complexity_router", - complexity_router_config={ - "classifier_type": "llm", - "classifier_llm_config": {"model": classifier, "timeout_ms": 30000}, - "tiers": {tier: target for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")}, - }, - ))) - key: Final = proxy.generate_key(KeyGenerateBody(models=[alias, classifier, target])) - stack.callback(proxy.delete_key, key) - yield AuditDeployment(alias, key) - - -class TestClassifierAudit: - @pytest.mark.covers( - "reliability.routing.classifier_audit.separates_provider_input_and_source", - exercised_on=("chat_completions", "messages", "responses"), - ) - @pytest.mark.parametrize("surface", ["chat_completions", "messages", "responses"]) - @pytest.mark.parametrize("provider", ["openai", "anthropic"]) - @pytest.mark.parametrize("redact", [False, True]) - def test_classifier_audit_separates_input_and_masked_source( - self, proxy: ProxyClient, audit_deployment: AuditDeployment, surface: Literal["chat_completions", "messages", "responses"], - redact: bool, - ) -> None: - marker: Final = unique_marker() - source_marker: Final = f"source-only-{marker}" - prompt: Final = f"Reply with hello. Request label {marker}" - metadata: Final = AuditMetadata(source_marker=source_marker) - body: Final = AuditRequest( - model=audit_deployment.alias, - messages=[ChatMessage(role="user", content=prompt)] if surface != "responses" else None, - input=prompt if surface == "responses" else None, - system="Be concise" if surface == "messages" else None, - instructions="Be concise" if surface == "responses" else None, - max_tokens=128 if surface != "responses" else None, - max_output_tokens=128 if surface == "responses" else None, - metadata=metadata if surface != "responses" else None, - litellm_metadata=metadata if surface == "responses" else None, - ) - path: Final = {"chat_completions": "/chat/completions", "messages": "/v1/messages", "responses": "/v1/responses"}[surface] - response: Final = unwrap(proxy.transport.post( - path, headers=AuditHeaders(authorization=f"Bearer {audit_deployment.key}", enable_redaction="true" if redact else None), - json=body, response_type=AuditResponse, - )) - assert response.id - rows: Final = proxy.poll_logs_for_key(audit_deployment.key, min_rows=2) - assert len(rows) == 2, "Expected a classifier spend row and a routed response spend row" - details: Final = tuple( - unwrap(proxy.transport.get( - f"/spend/logs/ui/{row.request_id}", headers=proxy.transport.master, - params=NoBody(), response_type=AuditDetail, - )) for row in rows - ) - adapter: Final = TypeAdapter(dict[str, JsonValue]) - requests: Final = tuple( - adapter.validate_json(detail.proxy_server_request) if isinstance(detail.proxy_server_request, str) - else detail.proxy_server_request or {} for detail in details - ) - audits: Final = tuple(item for item in requests if "classifier_input" in item) - if redact: - assert audits == () - assert all("originating_request_masked" not in item for item in requests) - return - assert len(audits) == 1, "The audit belongs only to the classifier call" - audit: Final = audits[0] - assert marker in json.dumps(audit["classifier_input"]) - assert source_marker not in json.dumps(audit["classifier_input"]) - assert source_marker in json.dumps(audit["originating_request_masked"]) - assert "synthetic-audit-secret" not in json.dumps(audit) - assert audit_deployment.alias in json.dumps(audit["originating_request_masked"]) - assert any("tier" in str(detail.response) for detail in details) From 208d554c008da91b74062413ad8d5d454f7746b5 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 13:04:36 -0700 Subject: [PATCH 053/126] fix(router): validate encrypted classifiers after deployment selection --- .../llms/base_llm/responses/transformation.py | 3 + .../llms/openai/responses/transformation.py | 3 + litellm/responses/main.py | 8 + .../complexity_router/README.md | 3 + .../complexity_router/complexity_router.py | 35 ++-- .../router_strategy/test_complexity_router.py | 151 ++++++++++++++++-- 6 files changed, 168 insertions(+), 35 deletions(-) diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 1365941fe2a..14f00aaaa21 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -62,6 +62,9 @@ class BaseResponsesAPIConfig(ABC): """ return False + def supports_encrypted_agent_messages(self) -> bool: + return False + def sign_request( self, headers: dict, diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 666e9b32011..833ae206024 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -110,6 +110,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): def supports_native_file_search(self) -> bool: return True + def supports_encrypted_agent_messages(self) -> bool: + return self.custom_llm_provider in (LlmProviders.OPENAI, LlmProviders.AZURE) + @staticmethod def _is_gpt_5_model(model: str) -> bool: """Return True only for actual OpenAI GPT-5 models. diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 7a210cdd970..e88cd618a8b 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -1078,6 +1078,7 @@ def responses( litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None) _is_async: Final = kwargs.pop("aresponses", False) is True skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False) + require_encrypted_task_support: Final = kwargs.pop("_require_encrypted_task_support", False) is True use_chat_completions_api = _pop_use_chat_completions_api_kw(kwargs) client_headers: Final = kwargs.get("headers") @@ -1186,6 +1187,13 @@ def responses( model, custom_llm_provider, deployment_model_info ) + if require_encrypted_task_support and ( + _bridges_to_chat_completions(responses_api_provider_config, use_chat_completions_api) + or responses_api_provider_config is None + or not responses_api_provider_config.supports_encrypted_agent_messages() + ): + raise ValueError("Encrypted task classification requires a compatible native Responses deployment") + local_vars.update(kwargs) # Map reasoning_effort (from litellm_params/proxy config) to reasoning when not set if reasoning is None and "reasoning_effort" in local_vars: diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 267def7cc5b..38dfd143cbc 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -367,6 +367,9 @@ local scoring shortcut in `heuristic_first` and `hybrid` modes. The configured c a native OpenAI or Azure OpenAI Responses deployment with access to the encrypted content. The provider handles the encrypted task, and the classifier still chooses the tier dynamically +Compatibility is checked after normal deployment selection. A paused incompatible member of the +classifier group does not prevent an eligible compatible deployment from classifying the task + Unsupported classifier deployments and provider decryption errors use the existing `classifier_fallback` policy. No fixed tier is introduced for encrypted tasks. Plaintext asks and requests carrying only historical encrypted reasoning retain the existing classifier path diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 77de4df3cd7..6df74b26f60 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -25,7 +25,7 @@ from threading import Lock from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast -from pydantic import BaseModel, TypeAdapter, create_model +from pydantic import BaseModel, TypeAdapter, ValidationError, create_model from litellm._logging import verbose_router_logger from litellm.constants import ( @@ -504,7 +504,10 @@ def _encrypted_classifier_task( raw_input: Final = (request_kwargs or EMPTY_MAPPING).get("input") if not isinstance(raw_input, list) or (request_kwargs or EMPTY_MAPPING).get("messages"): return None - items: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(raw_input) + try: + items: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(raw_input) + except ValidationError: + return None current: Final = next( ( item @@ -516,7 +519,10 @@ def _encrypted_classifier_task( ) if current is None or current.get("type") != "agent_message" or not isinstance(current.get("content"), list): return None - parts: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(current["content"]) + try: + parts: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(current["content"]) + except ValidationError: + return None if not any(part.get("type") == "encrypted_content" and part.get("encrypted_content") for part in parts): return None return { @@ -2058,7 +2064,7 @@ class ComplexityRouter(CustomLogger): classifier_call_params = MappingProxyType({"reasoning_effort": llm_config.reasoning_effort}) payload: Final = ( - self._native_classifier_payload(llm_config.model, messages_for_call, response_format, encrypted_task) + self._native_classifier_payload(messages_for_call, response_format, encrypted_task) if encrypted_task is not None else {"messages": messages_for_call, "response_format": response_format, **classifier_call_params} ) @@ -2098,7 +2104,6 @@ class ComplexityRouter(CustomLogger): def _native_classifier_payload( self, - model: str, messages: list[AllMessageValues], # mutable-ok: existing transformation accepts the SDK message list response_format: Mapping[str, object], encrypted_task: Mapping[str, object], @@ -2106,26 +2111,7 @@ class ComplexityRouter(CustomLogger): from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, ) - from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, get_llm_provider - from litellm.types.router import LiteLLM_Params - deployments: Final = self._group_deployments(model) - if not deployments: - raise ValueError("Encrypted task classification requires a native OpenAI Responses classifier deployment") - for params in (LiteLLM_Params.model_validate(deployment.get("litellm_params")) for deployment in deployments): - if declared_authenticating_provider(params.model, params.custom_llm_provider): - raise ValueError( - "Encrypted task classification requires a native OpenAI Responses classifier deployment" - ) - _, provider, _, _ = get_llm_provider(model=params.model, litellm_params=params) - if ( - provider not in ("openai", "azure") - or params.use_chat_completions_api - or params.model.startswith("openai/chat_completions/") - ): - raise ValueError( - "Encrypted task classification requires a native OpenAI Responses classifier deployment" - ) transformation: Final = LiteLLMResponsesTransformationHandler() input_items, instructions = transformation.convert_chat_completion_messages_to_responses_api(messages) llm_config: Final = self.config.classifier_llm_config @@ -2139,6 +2125,7 @@ class ComplexityRouter(CustomLogger): "instructions": instructions, "text": transformation.transform_response_format_to_text_format(dict(response_format)), "store": False, + "_require_encrypted_task_support": True, **reasoning, } diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 780465dad3b..2ea8a8b53fe 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -5,8 +5,10 @@ Tests the rule-based complexity scoring and tier assignment logic. """ import asyncio +from collections.abc import AsyncIterator import json from copy import deepcopy +from functools import partial import logging import sys import time @@ -14,6 +16,7 @@ from typing import Dict, Final, List from unittest.mock import AsyncMock, MagicMock, patch import pytest +import httpx from pydantic import ValidationError import litellm @@ -69,6 +72,7 @@ from litellm.types.router import ( TaggedPreRoutingStrategy, ) from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler requires_semantic_router = pytest.mark.skipif( @@ -2520,11 +2524,21 @@ def _native_classifier_router( classifier_type: str = "llm", deployment_model: str = "openai/gpt-6-astra", failure: Exception | None = None, + native_router: Router | None = None, + http_handler: AsyncHTTPHandler | None = None, ) -> tuple[ComplexityRouter, MagicMock]: dependency: Final = MagicMock( - aresponses=AsyncMock(return_value=_native_classifier_response(output), side_effect=failure), + aresponses=( + native_router.factory_function(partial(litellm.aresponses, client=http_handler), call_type="aresponses") + if native_router is not None + else AsyncMock(return_value=_native_classifier_response(output), side_effect=failure) + ), acompletion=AsyncMock(return_value=_llm_response('{"tier":"SIMPLE"}')), - get_model_list=MagicMock(return_value=[{"litellm_params": {"model": deployment_model}}]), + get_model_list=( + native_router.get_model_list + if native_router is not None + else MagicMock(return_value=[{"litellm_params": {"model": deployment_model}}]) + ), ) return ( ComplexityRouter( @@ -2533,7 +2547,11 @@ def _native_classifier_router( complexity_router_config={ "tiers": {"SIMPLE": "cheap-model", "REASONING": "deep-model"}, "classifier_type": classifier_type, - "classifier_llm_config": {"model": "classifier", "timeout_ms": 100, "reasoning_effort": "low"}, + "classifier_llm_config": { + "model": "classifier", + "timeout_ms": 5000 if native_router is not None else 100, + "reasoning_effort": "low", + }, "heuristic_first_max_tier": "SIMPLE" if classifier_type == "heuristic_first" else None, "hybrid_boundary_margin": 0.01 if classifier_type == "hybrid" else None, "classifier_fallback": "default_model", @@ -2546,6 +2564,18 @@ def _native_classifier_router( ) +@pytest.fixture +async def native_classifier_http() -> AsyncIterator[tuple[AsyncHTTPHandler, MagicMock]]: + respond: Final = MagicMock( + return_value=httpx.Response(200, json=_native_classifier_response('{"tier":"REASONING"}').model_dump()) + ) + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + handler.client = client + yield handler, respond + + class TestEncryptedTaskClassifier: @pytest.mark.asyncio @pytest.mark.parametrize("classifier_type", ["llm", "heuristic_first", "hybrid"]) @@ -2587,11 +2617,15 @@ class TestEncryptedTaskClassifier: assert "Caller constraints" not in call["instructions"] assert "SIMPLE" in call["instructions"] and "REASONING" in call["instructions"] assert call["text"]["format"]["schema"]["properties"]["tier"]["enum"] == [ - "SIMPLE", "MEDIUM", "COMPLEX", "REASONING" + "SIMPLE", + "MEDIUM", + "COMPLEX", + "REASONING", ] assert call["text"]["format"]["strict"] is True assert call["reasoning"] == {"effort": "low"} assert call["store"] is False + assert call["_require_encrypted_task_support"] is True assert call["stream"] is False assert "tools" not in call and "previous_response_id" not in call assert "messages" not in call and "response_format" not in call @@ -2606,13 +2640,25 @@ class TestEncryptedTaskClassifier: @pytest.mark.parametrize( "items", [ - [{"type": "reasoning", "encrypted_content": "opaque-history", "summary": []}, {"role": "user", "content": "hi"}], + [ + {"type": "reasoning", "encrypted_content": "opaque-history", "summary": []}, + {"role": "user", "content": "hi"}, + ], [_encrypted_agent_task(), {"role": "user", "content": "hi"}], [{**_encrypted_agent_task(), "content": [{"type": "input_text", "text": "hi"}]}], [{"role": "user", "content": "gAAAA is plain text"}], - [{"role": "user", "content": "hi"}, {"type": "function_call_output", "call_id": "call_1", "output": "opaque-provider-task"}], + [ + {"role": "user", "content": "hi"}, + {"type": "function_call_output", "call_id": "call_1", "output": "opaque-provider-task"}, + ], + ], + ids=[ + "historical-reasoning", + "older-encrypted-task", + "plaintext-agent", + "ciphertext-looking-text", + "tool-output", ], - ids=["historical-reasoning", "older-encrypted-task", "plaintext-agent", "ciphertext-looking-text", "tool-output"], ) async def test_other_asks_keep_chat_classifier(self, items: list[dict[str, object]]): router, dependency = _native_classifier_router() @@ -2639,9 +2685,28 @@ class TestEncryptedTaskClassifier: dependency.acompletion.assert_not_called() @pytest.mark.asyncio - @pytest.mark.parametrize("deployment_model", ["anthropic/test-classifier", "openai/chat_completions/gpt-6-astra"]) - async def test_incompatible_classifier_does_not_flatten_encryption(self, deployment_model: str): - router, dependency = _native_classifier_router(deployment_model=deployment_model) + @pytest.mark.parametrize( + "deployment_model", + ["anthropic/test-classifier", "openai/chat_completions/gpt-6-astra", "xai/test-classifier"], + ) + async def test_incompatible_classifier_does_not_flatten_encryption( + self, deployment_model: str, native_classifier_http: tuple[AsyncHTTPHandler, MagicMock] + ): + handler, respond = native_classifier_http + native: Final = Router( + model_list=[ + { + "model_name": "classifier", + "litellm_params": { + "model": deployment_model, + "api_key": "test-key", + "api_base": "https://classifier.test/v1", + }, + } + ], + num_retries=0, + ) + router, _ = _native_classifier_router(native_router=native, http_handler=handler) result: Final = await router.async_pre_routing_hook( model="encrypted-router", request_kwargs={"input": [_encrypted_agent_task()]} @@ -2649,8 +2714,72 @@ class TestEncryptedTaskClassifier: assert result.model == "deep-model" assert result.routing_decision["cause"] == "default_model_fallback" + respond.assert_not_called() + + @pytest.mark.asyncio + @pytest.mark.parametrize("blocked", [True, False]) + async def test_native_classifier_validates_selected_deployment( + self, blocked: bool, native_classifier_http: tuple[AsyncHTTPHandler, MagicMock] + ): + handler, respond = native_classifier_http + native: Final = Router( + model_list=[ + { + "model_name": "classifier", + "litellm_params": {"model": "anthropic/test-classifier", "api_key": "test-key", "order": 0}, + "model_info": {"id": "incompatible", "blocked": blocked}, + }, + { + "model_name": "classifier", + "litellm_params": { + "model": "openai/gpt-6-astra", + "api_key": "test-key", + "order": 1, + "api_base": "https://classifier.test/v1", + }, + "model_info": {"id": "compatible"}, + }, + ], + num_retries=0, + ) + router, _ = _native_classifier_router(native_router=native, http_handler=handler) + task: Final = _encrypted_agent_task() + + result: Final = await router.async_pre_routing_hook(model="encrypted-router", request_kwargs={"input": [task]}) + + assert result.model == "deep-model" + assert result.routing_decision["cause"] == ("llm_classifier" if blocked else "default_model_fallback") + if blocked: + respond.assert_called_once() + request: Final = respond.call_args.args[0] + assert request.url.path == "/v1/responses" + body: Final = json.loads(request.content) + assert body["input"][-1] == task + assert "_require_encrypted_task_support" not in body + else: + respond.assert_not_called() + + @pytest.mark.asyncio + @pytest.mark.parametrize("classifier_type", ["llm", "heuristic_first", "hybrid"]) + @pytest.mark.parametrize( + "input_items", + [ + ["unsupported-input-item"], + [{**_encrypted_agent_task(), "content": [{"type": "input_text", "text": "hi"}, None]}], + ], + ) + async def test_encrypted_detection_does_not_reject_other_input_shapes( + self, classifier_type: str, input_items: list[object] + ): + router, dependency = _native_classifier_router(classifier_type=classifier_type) + + result: Final = await router.aclassify("hi", request_kwargs={"input": input_items}) + + assert result.cause != "default_model_fallback" + assert result.tier == ComplexityTier.SIMPLE dependency.aresponses.assert_not_called() - dependency.acompletion.assert_not_called() + if classifier_type == "llm": + dependency.acompletion.assert_awaited_once() @pytest.mark.asyncio @pytest.mark.parametrize("failure", [ValueError("invalid_encrypted_content"), TimeoutError("classifier timed out")]) From 81f1778f535d3542e89e2b30eaf61b10a42cb4a8 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 13:05:58 -0700 Subject: [PATCH 054/126] feat(proxy): gate organization endpoints on an enterprise license The Admin UI and the docs already present Organizations as an enterprise feature, but every /organization route served unlicensed proxies. A router level dependency now enforces the license on all of them, and it resolves the auth dependency first so a bad key still gets 401 rather than 403. Claude-Session: https://claude.ai/code/session_01Se8ERtsqMQ3eVzWiLVMyNS --- .../organization_endpoints.py | 18 ++++++- .../test_organization_endpoints.py | 54 +++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index 1e711b036d2..96e946424bd 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -80,7 +80,23 @@ if TYPE_CHECKING: from prisma.models import LiteLLM_OrganizationTable as PrismaOrganizationTable from prisma.models import LiteLLM_UserTable as PrismaUserTable -router: Final = APIRouter() + +async def _enterprise_license_required( + _user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> None: + from litellm.proxy.proxy_server import premium_user + + if not premium_user: + raise HTTPException( + status_code=403, + detail={ + "error": "Organizations are only available for LiteLLM Enterprise users. " + f"{CommonProxyErrors.not_premium_user.value}" + }, + ) + + +router: Final = APIRouter(dependencies=[Depends(_enterprise_license_required)]) class _ObjectPermissionRow(Protocol): diff --git a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py index 4d13e054e46..328030a17f8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py @@ -1226,3 +1226,57 @@ def test_v2_update_organization_is_in_openapi_schema(): v2_path = app.openapi()["paths"]["/v2/organization/{organization_id}"] assert v2_path["patch"]["tags"] == ["organization management"] assert "OrganizationUpdateRequestV2" in json.dumps(v2_path["patch"]["requestBody"]) + + +def _organization_route_targets() -> list[tuple[str, str]]: + from fastapi.routing import APIRoute + + from litellm.proxy.management_endpoints.organization_endpoints import router + + return [ + (method, route.path.replace("{organization_id}", "org-under-test")) + for route in router.routes + if isinstance(route, APIRoute) + for method in sorted(route.methods - {"HEAD", "OPTIONS"}) + ] + + +def _organization_test_client() -> TestClient: + from fastapi import FastAPI + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.management_endpoints.organization_endpoints import router + + app = FastAPI() + app.include_router(router) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="sk-test", user_role=LitellmUserRoles.PROXY_ADMIN + ) + return TestClient(app, raise_server_exceptions=False) + + +@pytest.mark.parametrize(("method", "path"), _organization_route_targets()) +def test_organization_routes_are_blocked_without_enterprise_license(monkeypatch, method, path): + """Every /organization route is enterprise-only, even for a proxy admin.""" + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "premium_user", False, raising=False) + + response = _organization_test_client().request(method, path, json={}) + + assert response.status_code == 403 + assert "Organizations" in response.json()["detail"]["error"] + + +@pytest.mark.parametrize(("method", "path"), _organization_route_targets()) +def test_organization_routes_pass_the_license_gate_with_enterprise_license(monkeypatch, method, path): + """With a license the gate is transparent: whatever fails next, it is not the license check.""" + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "premium_user", True, raising=False) + monkeypatch.setattr(proxy_server, "prisma_client", None, raising=False) + + response = _organization_test_client().request(method, path, json={}) + + assert "LiteLLM Enterprise" not in response.text From 55ab5ee53d48ef165886de31144b1e172f8208e1 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:08:46 -0700 Subject: [PATCH 055/126] fix(mcp): preserve identity checks across cached OAuth credentials --- AGENTS.md | 2 + litellm/proxy/_experimental/mcp_server/db.py | 34 +++++--- .../mcp_server/oauth2_token_cache.py | 25 ++++-- .../mcp_server/oauth_identity_binding.py | 2 +- .../authz_code_refresher.py | 11 +++ .../outbound_credentials/oauth_token_store.py | 1 + .../per_user_oauth_store.py | 33 ++++++-- .../outbound_credentials/token_cache_codec.py | 36 ++++++-- .../outbound_credentials/v2_token_store.py | 2 + .../test_authz_code_refresher.py | 18 ++-- .../test_per_user_oauth_store.py | 82 +++++++++++++++++-- .../test_token_cache_codec.py | 14 ++++ .../mcp_server/test_db_credentials.py | 75 +++++++++++++++++ .../mcp_server/test_oauth_identity_binding.py | 30 +++++-- 14 files changed, 311 insertions(+), 54 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 41921fdff4d..a1e8f6f618d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1 +1,3 @@ Read @CLAUDE.md for coding guidelines + +Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review; never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or lint budgets to satisfy a check diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 08f83aeb756..789b2ffaef4 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -6,6 +6,7 @@ from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, cast +from fastapi import HTTPException from typing_extensions import ReadOnly from litellm._logging import verbose_proxy_logger @@ -1692,13 +1693,18 @@ async def refresh_user_oauth_token( ) return None - binding_proof: Final = await enforce_oauth_identity_binding( - server=server, - token_response=body, - litellm_user_id=user_id, - grant_type="refresh_token", - refresh_ownership=RefreshTokenPresented(refresh_token), - ) + try: + binding_proof: Final = await enforce_oauth_identity_binding( + server=server, + token_response=body, + litellm_user_id=user_id, + grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented(refresh_token), + ) + except HTTPException as exc: + if exc.status_code != 403: + raise + return None access_token: Final[str | None] = body.get("access_token") if not access_token: @@ -1812,8 +1818,14 @@ async def resolve_user_oauth_access_token( binding: Final = server.oauth_identity_binding enforce_binding: Final = binding is not None and binding.mode == "enforce" - if enforce_binding: - await mcp_per_user_token_cache.delete(user_id, server_id) + if prefetched_creds is None and enforce_binding and binding is not None: + bound_token: Final = await mcp_per_user_token_cache.get_token(user_id, server_id) + if bound_token is not None: + if await credential_binding_matches( + binding, user_id, server_id, {"identity_binding_proof": bound_token.identity_binding_proof} + ): + return bound_token.access_token + await mcp_per_user_token_cache.delete(user_id, server_id) if prefetched_creds is None and not enforce_binding: cached_token: Final = await mcp_per_user_token_cache.get(user_id, server_id) if cached_token is not None: @@ -1848,7 +1860,9 @@ async def resolve_user_oauth_access_token( access_token: Final[str] = cred["access_token"] if prefetched_creds is None: ttl: Final = _compute_per_user_token_ttl(server, _remaining_token_seconds(cred.get("expires_at"))) - await mcp_per_user_token_cache.set(user_id, server_id, access_token, ttl) + await mcp_per_user_token_cache.set( + user_id, server_id, access_token, ttl, identity_binding_proof=cred.get("identity_binding_proof") + ) return access_token except Exception as e: verbose_proxy_logger.warning( diff --git a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py index a4ef970b87a..42edc2999ab 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py +++ b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py @@ -28,6 +28,8 @@ from litellm.proxy._experimental.mcp_server.oauth_utils import ( build_upstream_oauth2_token_request, resolve_upstream_resource, ) +from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import OAuthToken +from litellm.proxy._experimental.mcp_server.outbound_credentials.token_cache_codec import OAuthTokenCacheCodec from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, @@ -233,8 +235,17 @@ class MCPPerUserTokenCache: def _cache_key(self, user_id: str, server_id: str) -> str: return f"{MCP_PER_USER_TOKEN_REDIS_KEY_PREFIX}:{user_id}:{server_id}" + def _codec(self) -> OAuthTokenCacheCodec: + return OAuthTokenCacheCodec( + encrypt_value_helper, + lambda blob: decrypt_value_helper(blob, key="mcp_per_user_token", exception_type="debug"), + ) + async def get(self, user_id: str, server_id: str) -> str | None: - """Return the plaintext access_token, or None on miss/error.""" + token: Final = await self.get_token(user_id, server_id) + return token.access_token if token is not None else None + + async def get_token(self, user_id: str, server_id: str) -> OAuthToken | None: try: from litellm.proxy.proxy_server import user_api_key_cache # noqa: PLC0415 @@ -242,12 +253,7 @@ class MCPPerUserTokenCache: encrypted: Final = await user_api_key_cache.async_get_cache(key) if encrypted is None: return None - plaintext: Final = decrypt_value_helper( - encrypted, - key="mcp_per_user_token", - exception_type="debug", - ) - return plaintext or None + return self._codec().decode(encrypted) except Exception as exc: verbose_logger.debug( "MCPPerUserTokenCache.get failed for user=%s server=%s: %s", @@ -263,13 +269,16 @@ class MCPPerUserTokenCache: server_id: str, access_token: str, ttl: int, + identity_binding_proof: str | None = None, ) -> None: """Store NaCl-encrypted access_token in Redis with the given TTL.""" try: from litellm.proxy.proxy_server import user_api_key_cache # noqa: PLC0415 key: Final = self._cache_key(user_id, server_id) - encrypted: Final = encrypt_value_helper(access_token) + encrypted: Final = self._codec().encode( + OAuthToken(access_token=access_token, identity_binding_proof=identity_binding_proof) + ) await user_api_key_cache.async_set_cache(key, encrypted, ttl=ttl) verbose_logger.debug( "MCPPerUserTokenCache.set: cached token for user=%s server=%s ttl=%ds", diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py index 7ca9e6a23df..f02e6c85d9b 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -342,7 +342,7 @@ async def _evaluate_binding( claims: Final = _decode_id_token(id_token, binding, signing_key) if isinstance(claims, _BindingRejection): return claims - if grant_type == "authorization_code": + if grant_type == "authorization_code" and (binding.mode == "enforce" or expected_nonce is not None): nonce: Final = claims.get("nonce") if not expected_nonce or not isinstance(nonce, str) or not hmac.compare_digest(nonce, expected_nonce): return _BindingRejection( diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py index 824459fd9b9..af1e82eab82 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py @@ -14,6 +14,8 @@ import time from collections.abc import Awaitable, Callable from typing import TYPE_CHECKING, Final, Protocol +from fastapi import HTTPException + from litellm._logging import verbose_logger from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( TokenEndpointAuthConfigError, @@ -93,6 +95,14 @@ class AuthorizationCodeRefresher: self._identity_validator = identity_validator async def refresh(self, user_id: str, server_id: str, token: OAuthToken) -> OAuthToken | None: + try: + return await self._refresh(user_id, server_id, token) + except HTTPException as exc: + if exc.status_code != 403: + raise + return None + + async def _refresh(self, user_id: str, server_id: str, token: OAuthToken) -> OAuthToken | None: if token.refresh_token is None: return None server: Final = self._server_lookup(server_id) @@ -162,4 +172,5 @@ class AuthorizationCodeRefresher: expires_at=self._clock() + expires_in if expires_in is not None else None, refresh_token=new_refresh, scopes=scopes, + identity_binding_proof=binding_proof, ) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/oauth_token_store.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/oauth_token_store.py index e0cd9e8e5ed..0ac4296f498 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/oauth_token_store.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/oauth_token_store.py @@ -41,6 +41,7 @@ class OAuthToken: expires_at: float | None = None refresh_token: str | None = None scopes: tuple[str, ...] = () + identity_binding_proof: str | None = None def __repr__(self) -> str: has_refresh: Final = self.refresh_token is not None diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py index efcc613c539..2897c3e8e4a 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py @@ -12,6 +12,7 @@ from __future__ import annotations import asyncio from collections.abc import Callable, Mapping +from functools import partial from typing import TYPE_CHECKING, Final from litellm._logging import verbose_logger @@ -39,7 +40,6 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.token_cache_cod OAuthTokenCacheCodec, ) from litellm.proxy._experimental.mcp_server.outbound_credentials.v2_token_store import ( - CredentialReader, V2PerUserTokenStore, ) @@ -146,12 +146,26 @@ def _runtime_backend_and_coordinator() -> tuple[TokenCacheBackend | None, Refres return backend, coordinator, True +async def _read_bound_credential( + server_lookup: ServerLookup, user_id: str, server_id: str +) -> Mapping[str, object] | None: + credential: Final = await _read_credential(user_id, server_id) + server: Final = server_lookup(server_id) + binding: Final = server.oauth_identity_binding if server else None + if credential is not None and binding is not None and binding.mode == "enforce": + if not await credential_binding_matches(binding, user_id, server_id, credential): + return None + return credential + + def _build_per_user_oauth_token_store( server_lookup: ServerLookup, ) -> tuple[CachedOAuthTokenStore, bool]: backend, coordinator, uses_redis = _runtime_backend_and_coordinator() refresher: Final = AuthorizationCodeRefresher(server_lookup, _post_token_endpoint, _persist_credential) - refreshing: Final = RefreshingTokenStore(V2PerUserTokenStore(_read_credential), refresher, coordinator=coordinator) + refreshing: Final = RefreshingTokenStore( + V2PerUserTokenStore(partial(_read_bound_credential, server_lookup)), refresher, coordinator=coordinator + ) return CachedOAuthTokenStore(refreshing, default_ttl_seconds=_DEFAULT_TTL_SECONDS, backend=backend), uses_redis @@ -176,10 +190,8 @@ class LazyPerUserOAuthTokenStore: *, store_builder: StoreBuilder = _build_per_user_oauth_token_store, redis_available: Callable[[], bool] = _redis_cache_is_available, - credential_reader: CredentialReader = _read_credential, ) -> None: self._server_lookup = server_lookup - self._credential_reader = credential_reader self._store_builder = store_builder self._redis_available = redis_available self._store: InvalidatableOAuthTokenStore | None = None @@ -188,13 +200,18 @@ class LazyPerUserOAuthTokenStore: self._local_fetches = 0 async def fetch(self, user_id: str, server_id: str) -> OAuthToken | None: + token: Final = await self._fetch_token(user_id, server_id) server: Final = self._server_lookup(server_id) binding: Final = server.oauth_identity_binding if server else None - if binding is not None and binding.mode == "enforce": - credential: Final = await self._credential_reader(user_id, server_id) - await self.invalidate(user_id, server_id) - if credential is None or not await credential_binding_matches(binding, user_id, server_id, credential): + if token is not None and binding is not None and binding.mode == "enforce": + if not await credential_binding_matches( + binding, user_id, server_id, {"identity_binding_proof": token.identity_binding_proof} + ): + await self.invalidate(user_id, server_id) return None + return token + + async def _fetch_token(self, user_id: str, server_id: str) -> OAuthToken | None: if self._uses_redis: store = self._store if store is not None: diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_cache_codec.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_cache_codec.py index 14cc309b685..56dc3f19cdd 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_cache_codec.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_cache_codec.py @@ -1,24 +1,26 @@ """Serialize + encrypt boundary for caching an OAuth token in a shared (Redis) cache. -A cross-replica cache must serialize the token, and a plaintext bearer in Redis is a leak, so this -encrypts the value (NaCl in production via the injected ``encrypt``, identity in tests). It caches -**only** the ``access_token``: the hot path needs just the bearer, expiry is carried by the cache -entry's TTL (set from the token's ``expires_at`` by the cache), and the long-lived refresh_token stays -in the DB - the refresh path is always a cache miss that re-reads it - so it never reaches Redis. A -decoded token therefore carries only the bearer (``expires_at`` and ``refresh_token`` both None); the -TTL, not the value, bounds its life. An empty/undecryptable blob (e.g. master-key rotation) is a miss. +Shared cache values contain an encrypted access token and optional identity-binding proof. +Refresh tokens remain in the database; cache TTL bounds the access token's lifetime. +Legacy bearer-only entries decode without proof and cannot satisfy identity enforcement. """ from __future__ import annotations +import json from collections.abc import Callable from dataclasses import dataclass from typing import Final +from pydantic import TypeAdapter, ValidationError + from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) +_BOUND_PREFIX: Final = "litellm-bound-oauth-v1:" +_BOUND_PAYLOAD: Final = TypeAdapter(dict[str, str]) + @dataclass(frozen=True, slots=True) class OAuthTokenCacheCodec: @@ -26,10 +28,30 @@ class OAuthTokenCacheCodec: decrypt: Callable[[str], str | None] def encode(self, token: OAuthToken) -> str: + if token.identity_binding_proof is not None: + return self.encrypt( + _BOUND_PREFIX + + json.dumps( + { + "access_token": token.access_token, + "identity_binding_proof": token.identity_binding_proof, + } + ) + ) return self.encrypt(token.access_token) def decode(self, blob: str) -> OAuthToken | None: access_token: Final = self.decrypt(blob) if not access_token: return None + if access_token.startswith(_BOUND_PREFIX): + try: + payload: Final = _BOUND_PAYLOAD.validate_json(access_token[len(_BOUND_PREFIX) :]) + except ValidationError: + return None + bearer: Final = payload.get("access_token") + proof: Final = payload.get("identity_binding_proof") + if not bearer or not proof: + return None + return OAuthToken(access_token=bearer, identity_binding_proof=proof) return OAuthToken(access_token=access_token, refresh_token=None) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/v2_token_store.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/v2_token_store.py index 4d88ec8b025..0f18931d118 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/v2_token_store.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/v2_token_store.py @@ -46,11 +46,13 @@ def _to_oauth_token(payload: Mapping[str, object]) -> OAuthToken | None: return None refresh_token: Final = payload.get("refresh_token") expires_at: Final = payload.get("expires_at") + binding_proof: Final = payload.get("identity_binding_proof") return OAuthToken( access_token=access_token, expires_at=_iso_to_epoch(expires_at) if isinstance(expires_at, str) else None, refresh_token=refresh_token if isinstance(refresh_token, str) else None, scopes=_to_scopes(payload.get("scopes")), + identity_binding_proof=binding_proof if isinstance(binding_proof, str) else None, ) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py index 16707f6ace8..df068b60338 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py @@ -293,17 +293,18 @@ async def test_refresh_uses_admin_entered_token_url_when_issuer_yield_empties_re @pytest.mark.asyncio async def test_identity_rejection_never_persists_or_returns_refreshed_token(): from unittest.mock import AsyncMock + from fastapi import HTTPException validator = AsyncMock(side_effect=HTTPException(status_code=403, detail="oauth_principal_mismatch")) persist = AsyncMock() refresher = AuthorizationCodeRefresher( - _lookup(_Server()), _endpoint({"access_token": "foreign-token", "id_token": "foreign-identity"}), - persist, identity_validator=validator, + _lookup(_Server()), + _endpoint({"access_token": "foreign-token", "id_token": "foreign-identity"}), + persist, + identity_validator=validator, ) - with pytest.raises(HTTPException) as error: - await refresher.refresh("alice", "srv", OAuthToken(access_token="old", refresh_token="old-rt")) - assert error.value.status_code == 403 + assert await refresher.refresh("alice", "srv", OAuthToken(access_token="old", refresh_token="old-rt")) is None persist.assert_not_awaited() @@ -314,10 +315,13 @@ async def test_verified_refresh_preserves_binding_proof_in_storage(): validator = AsyncMock(return_value="verified-binding") persist = AsyncMock() refresher = AuthorizationCodeRefresher( - _lookup(_Server()), _endpoint({"access_token": "new", "refresh_token": "rotated"}), - persist, identity_validator=validator, + _lookup(_Server()), + _endpoint({"access_token": "new", "refresh_token": "rotated"}), + persist, + identity_validator=validator, ) token = await refresher.refresh("alice", "srv", OAuthToken(access_token="old", refresh_token="old-rt")) assert token.access_token == "new" assert token.refresh_token == "rotated" + assert token.identity_binding_proof == "verified-binding" assert persist.await_args.kwargs["identity_binding_proof"] == "verified-binding" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py index bd190b09c82..e7308884060 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py @@ -254,20 +254,86 @@ async def test_enforcement_invalidates_cached_legacy_credentials_before_use(): from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer server = MCPServer( - server_id="srv", name="srv", transport=MCPTransport.http, auth_type=MCPAuth.oauth2, + server_id="srv", + name="srv", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, oauth_identity_binding=MCPOAuthIdentityBinding( - mode="enforce", issuer="https://idp.example.com", audiences=["client"], + mode="enforce", + issuer="https://idp.example.com", + audiences=["client"], ), ) cached = _RecordingStore("belongs-to-bob") - async def read_legacy(user_id, server_id): - return {"access_token": "belongs-to-bob", "refresh_token": "bobs-refresh"} - store = LazyPerUserOAuthTokenStore( - lambda server_id: server, store_builder=lambda lookup: (cached, False), - redis_available=lambda: False, credential_reader=read_legacy, + lambda server_id: server, + store_builder=lambda lookup: (cached, False), + redis_available=lambda: False, ) assert await store.fetch("alice", "srv") is None - assert cached.calls == [] + assert cached.calls == [("alice", "srv")] assert cached.invalidations == [("alice", "srv")] + + +@pytest.mark.asyncio +async def test_enforced_cache_hit_avoids_credential_read_and_rejects_changed_policy(monkeypatch): + from unittest.mock import AsyncMock + + from litellm.proxy._experimental.mcp_server.oauth_identity_binding import current_binding_proof + from litellm.proxy._experimental.mcp_server.outbound_credentials import per_user_oauth_store as module + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="srv", + name="srv", + transport="http", + auth_type="oauth2", + oauth_identity_binding={ + "mode": "enforce", + "issuer": "https://idp.example", + "audiences": ["client"], + "caller_field": "user_id", + "principal_claim": "sub", + }, + ) + proof = await current_binding_proof(server.oauth_identity_binding, "alice", "srv") + read = AsyncMock(return_value={"access_token": "alice-token", "identity_binding_proof": proof}) + monkeypatch.setattr(module, "_read_credential", read) + monkeypatch.setattr(module, "_runtime_backend_and_coordinator", lambda: (None, None, False)) + store = LazyPerUserOAuthTokenStore(lambda _: server, redis_available=lambda: False) + assert (await store.fetch("alice", "srv")).access_token == "alice-token" + assert (await store.fetch("alice", "srv")).access_token == "alice-token" + read.assert_awaited_once_with("alice", "srv") + server.oauth_identity_binding = server.oauth_identity_binding.model_copy(update={"audiences": ["changed"]}) + assert await store.fetch("alice", "srv") is None + read.assert_awaited_once() + assert await store.fetch("alice", "srv") is None + assert read.await_count == 2 + + +@pytest.mark.asyncio +async def test_expired_unverified_credential_never_reaches_refresh(monkeypatch): + from unittest.mock import AsyncMock + + from litellm.proxy._experimental.mcp_server.outbound_credentials import per_user_oauth_store as module + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="srv", + name="srv", + transport="http", + auth_type="oauth2", + token_url="https://idp.example/token", + oauth_identity_binding={"mode": "enforce", "issuer": "https://idp.example", "audiences": ["client"]}, + ) + read = AsyncMock( + return_value={"access_token": "bob", "refresh_token": "bob-refresh", "expires_at": "2000-01-01T00:00:00Z"} + ) + post = AsyncMock() + monkeypatch.setattr(module, "_read_credential", read) + monkeypatch.setattr(module, "_post_token_endpoint", post) + monkeypatch.setattr(module, "_runtime_backend_and_coordinator", lambda: (None, None, False)) + store = LazyPerUserOAuthTokenStore(lambda _: server, redis_available=lambda: False) + assert await store.fetch("alice", "srv") is None + post.assert_not_awaited() diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_cache_codec.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_cache_codec.py index 17a7e13af03..12a04c4b4dc 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_cache_codec.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_cache_codec.py @@ -52,3 +52,17 @@ def test_undecryptable_blob_is_a_miss(): def test_empty_plaintext_is_a_miss(): codec = OAuthTokenCacheCodec(encrypt=lambda s: s, decrypt=lambda b: b) assert codec.decode("") is None + + +def test_bound_token_round_trip_preserves_proof_without_refresh_secret(): + codec = _wrapping_codec() + blob = codec.encode(OAuthToken("alice-token", refresh_token="private-refresh", identity_binding_proof="proof")) + assert "private-refresh" not in blob + decoded = codec.decode(blob) + assert decoded == OAuthToken("alice-token", identity_binding_proof="proof") + + +def test_malformed_bound_entries_fail_closed(): + codec = _wrapping_codec() + for payload in ("not-json", "{}", '{"access_token":"at"}', '{"access_token":1,"identity_binding_proof":"p"}'): + assert codec.decode("enc:litellm-bound-oauth-v1:" + payload) is None diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py index 8e2adbeb06e..60a5e1a22bb 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py @@ -1629,3 +1629,78 @@ async def test_enforcement_rejects_preexisting_unverified_credential(): cred={"access_token": "belongs-to-bob", "refresh_token": "bobs-refresh-token"}, ) assert result is None + + +@pytest.mark.asyncio +async def test_refresh_identity_rejection_returns_reauthentication_without_persisting(monkeypatch): + from fastapi import HTTPException + from litellm.proxy._experimental.mcp_server import db as module + + validator = AsyncMock(side_effect=HTTPException(status_code=403, detail="oauth_principal_mismatch")) + monkeypatch.setattr(module, "enforce_oauth_identity_binding", validator) + result, captured = await _run_refresh( + monkeypatch, _refresh_server(), {"access_token": "bob", "refresh_token": "rotated"} + ) + assert result is None + assert captured["data"]["grant_type"] == "refresh_token" + module.store_user_oauth_credential.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_verified_legacy_cache_reads_avoid_database_and_reject_policy_changes(monkeypatch): + from litellm.caching.caching import DualCache + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import db as module + from litellm.proxy._experimental.mcp_server.oauth2_token_cache import mcp_per_user_token_cache + from litellm.proxy._experimental.mcp_server.oauth_identity_binding import current_binding_proof + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="srv", + name="srv", + transport="http", + auth_type="oauth2", + oauth_identity_binding={ + "mode": "enforce", + "issuer": "https://idp.example", + "audiences": ["client"], + "caller_field": "user_id", + "principal_claim": "sub", + }, + ) + proof = await current_binding_proof(server.oauth_identity_binding, "alice", "srv") + monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache()) + read = AsyncMock(return_value=None) + monkeypatch.setattr(module, "get_user_oauth_credential", read) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + await mcp_per_user_token_cache.set("alice", "srv", "alice-token", 60, identity_binding_proof=proof) + assert await module.resolve_user_oauth_access_token("alice", server) == "alice-token" + assert await module.resolve_user_oauth_access_token("alice", server) == "alice-token" + read.assert_not_awaited() + server.oauth_identity_binding = server.oauth_identity_binding.model_copy(update={"audiences": ["changed"]}) + assert await module.resolve_user_oauth_access_token("alice", server) is None + read.assert_awaited_once() + assert await mcp_per_user_token_cache.get_token("alice", "srv") is None + + +@pytest.mark.asyncio +async def test_unverified_legacy_cache_cannot_bypass_enforcement(monkeypatch): + from litellm.caching.caching import DualCache + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import db as module + from litellm.proxy._experimental.mcp_server.oauth2_token_cache import mcp_per_user_token_cache + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="srv", + name="srv", + transport="http", + auth_type="oauth2", + oauth_identity_binding={"mode": "enforce", "issuer": "https://idp.example", "audiences": ["client"]}, + ) + monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache()) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(module, "get_user_oauth_credential", AsyncMock(return_value={"access_token": "bob"})) + await mcp_per_user_token_cache.set("alice", "srv", "bob", 60) + assert await module.resolve_user_oauth_access_token("alice", server) is None + assert await mcp_per_user_token_cache.get("alice", "srv") is None diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py index 6f291865e20..0036035f448 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py @@ -13,14 +13,14 @@ from pydantic import ValidationError from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( RefreshOwnershipProven, - VerifiedRefreshToken, RefreshTokenPresented, - current_binding_proof, + VerifiedRefreshToken, _discover_jwks_url, _fetch_issuer_jwks, _load_caller_principal, _load_stored_refresh_token, _select_signing_key, + current_binding_proof, enforce_oauth_identity_binding, ) from litellm.types.mcp import MCPAuth, MCPTransport @@ -471,19 +471,20 @@ async def test_refresh_with_mismatched_id_token_rejected(): @pytest.mark.asyncio -async def test_audit_mode_logs_but_does_not_reject(): +async def test_audit_mode_logs_but_does_not_reject(caplog): token: Final = _sign_id_token({"email": "mallory@example.com", "email_verified": True}) result: Final = await enforce_oauth_identity_binding( server=_server(mode="audit"), token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", - expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) assert result is None + assert "oauth_principal_mismatch" in caplog.text + assert "nonce" not in caplog.text @pytest.mark.asyncio @@ -642,6 +643,25 @@ async def test_binding_proof_rejects_changed_user_or_policy(): def test_identity_binding_rejects_modes_without_gateway_credential_custody(auth_type): with pytest.raises(ValidationError, match="gateway-managed per-user"): MCPServer( - server_id="srv", name="srv", transport=MCPTransport.http, auth_type=auth_type, + server_id="srv", + name="srv", + transport=MCPTransport.http, + auth_type=auth_type, oauth_identity_binding=_server().oauth_identity_binding, ) + + +@pytest.mark.asyncio +async def test_audit_matching_login_without_nonce_does_not_report_failure(caplog): + token: Final = _sign_id_token({"email": "alice@example.com", "email_verified": True}) + result: Final = await enforce_oauth_identity_binding( + server=_server(mode="audit"), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert result is None + assert "oauth_identity_binding audit" not in caplog.text From c2176c786e0fc1fed274ade46bbf901fa33c34a3 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 13:12:04 -0700 Subject: [PATCH 056/126] fix(router): redact cookies and preserve legacy classifier logs --- .../sensitive_data_masker.py | 1 + .../test_classifier_logging.py | 13 +++++ .../test_sensitive_data_masker.py | 9 +++- ui/litellm-dashboard/eslint-suppressions.json | 5 -- ... => LogDetailContent.integration.test.tsx} | 51 +++++++++++++++++++ .../LogDetailsDrawer/LogDetailContent.tsx | 15 +++--- 6 files changed, 80 insertions(+), 14 deletions(-) rename ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/{LogDetailContent.test.tsx => LogDetailContent.integration.test.tsx} (89%) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 22dd4170963..b4c1beea33e 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -15,6 +15,7 @@ _DEFAULT_SENSITIVE_PATTERNS: Final = frozenset( "token", "auth", "authorization", + "cookie", "credential", # Plural form: Vertex uses ``vertex_credentials``; segment-exact # matching otherwise misses it because "credential" != "credentials". diff --git a/tests/test_litellm/litellm_core_utils/test_classifier_logging.py b/tests/test_litellm/litellm_core_utils/test_classifier_logging.py index c923225c750..ccf14ff06c7 100644 --- a/tests/test_litellm/litellm_core_utils/test_classifier_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_classifier_logging.py @@ -33,6 +33,19 @@ def test_originating_snapshot_masks_nested_credentials_without_altering_source() assert body["metadata"]["nested"][0]["Authorization"] == "Bearer secret" +@pytest.mark.parametrize("header", ["Cookie", "cookie", "COOKIE", "sEt-CoOkIe"]) +def test_originating_snapshot_redacts_cookie_headers_shared_with_caller_metadata(header: str) -> None: + headers: Final = {header: "session=synthetic-session-credential", "content-type": "application/json"} + body: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {"headers": headers}} + snapshot: Final = masked_originating_request({"proxy_server_request": {"body": body, "headers": headers}}) + assert snapshot == { + "messages": [{"role": "user", "content": "hello"}], + "metadata": {"headers": {header: "REDACTED", "content-type": "application/json"}}, + } + assert headers[header] == "session=synthetic-session-credential" + assert body["metadata"]["headers"][header] == "session=synthetic-session-credential" + + @pytest.mark.parametrize("value", [None, "not-json", [], {"messages": object()}]) def test_invalid_provider_payload_is_not_reported_as_captured(value: object) -> None: assert classifier_input_snapshot(value) is None diff --git a/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py b/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py index c8b6772d965..1551c3fd6e6 100644 --- a/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py +++ b/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py @@ -358,7 +358,12 @@ def test_redact_credentials_in_payload_leaves_no_fragment_of_the_secret(): "azure_ad_token": fake_token, "aws_secret_access_key": "fake-aws-secret-0000", "vertex_credentials": {"private_key": "fake-pem"}, - "extra_headers": {"Authorization": "Bearer fake-bearer-0000", "x-request-id": "abc123"}, + "extra_headers": { + "Authorization": "Bearer fake-bearer-0000", + "Cookie": "session=fake-session", + "Set-Cookie": "session=fake-session; HttpOnly", + "x-request-id": "abc123", + }, "model": "gpt-4o-mini", "max_tokens": 17, "temperature": 0.25, @@ -373,6 +378,8 @@ def test_redact_credentials_in_payload_leaves_no_fragment_of_the_secret(): assert "fake-bearer-0000" not in str(result) assert result["api_key"] == "REDACTED" assert result["extra_headers"]["Authorization"] == "REDACTED" + assert result["extra_headers"]["Cookie"] == "REDACTED" + assert result["extra_headers"]["Set-Cookie"] == "REDACTED" assert result["extra_headers"]["x-request-id"] == "abc123" assert result["model"] == "gpt-4o-mini" assert result["max_tokens"] == 17 diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 76ac60a6453..e2a08a40bcb 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -2325,11 +2325,6 @@ "count": 4 } }, - "src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx": { - "no-nested-ternary": { - "count": 3 - } - }, "src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx": { "no-nested-ternary": { "count": 2 diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx similarity index 89% rename from ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx rename to ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx index e0d6a1d41e0..d538b16d6a2 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx @@ -272,6 +272,57 @@ describe("LogDetailContent", () => { expect(screen.queryByRole("tab", { name: "Request" })).not.toBeInTheDocument(); }); + it.each(["object", "serialized", "messages only", "null captures"])( + "preserves request inspection and copying for classifier logs with %s data", + async (shape) => { + const user = userEvent.setup(); + const messages = [{ role: "user", content: "legacy classifier prompt" }]; + const request = { + messages, + temperature: 0.5, + ...(shape === "null captures" ? { classifier_input: null, originating_request_masked: null } : {}), + }; + const storedRequest = shape === "serialized" ? JSON.stringify(request) : request; + const logEntry: Partial = { + call_type: "acompletion", + messages, + proxy_server_request: shape === "messages only" ? undefined : storedRequest, + metadata: { status: "success", internal_call_origin: "autorouter_classifier" }, + }; + render(); + + expect(screen.getByText("legacy classifier prompt")).toBeInTheDocument(); + expect(screen.queryByRole("region", { name: "Classifier input" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("tab", { name: "JSON", exact: true })); + await user.click(screen.getByRole("button", { name: "Copy JSON", exact: true })); + expect(await navigator.clipboard.readText()).toBe( + JSON.stringify(shape === "messages only" ? messages : request, null, 2), + ); + }, + ); + + it.each(["classifier_input", "originating_request_masked"])( + "shows partial classifier audits when only %s is captured", + (field) => { + render( + , + ); + + expect(screen.getByRole("region", { name: "Classifier input" })).toBeInTheDocument(); + expect(screen.getByRole("region", { name: "Originating request, credentials masked" })).toBeInTheDocument(); + expect(screen.getByText("Not captured or message logging disabled")).toBeInTheDocument(); + expect(screen.queryByText("Request & Response")).not.toBeInTheDocument(); + }, + ); + it("should display Request and Response tabs when JSON view is selected", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index e29f2810926..69be5425ea1 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -73,6 +73,9 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok const isClassifier = metadata.internal_call_origin === AUTOROUTER_CLASSIFIER_ORIGIN && (logEntry.call_type === "completion" || logEntry.call_type === "acompletion"); + const rawRequest = formatData(logEntry.proxy_server_request || logEntry.messages); + const hasClassifierAudit = + isClassifier && (rawRequest?.classifier_input != null || rawRequest?.originating_request_masked != null); const hasMessages = checkHasMessages(logEntry.messages); const hasResponse = checkHasResponse(logEntry.response); @@ -93,10 +96,6 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok // Vector store data const hasVectorStoreData = checkHasVectorStoreData(metadata); - const getRawRequest = () => { - return formatData(logEntry.proxy_server_request || logEntry.messages); - }; - const getFormattedResponse = () => { if (hasError && errorInfo) { return { @@ -202,14 +201,14 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok
) : null} - {!isLoadingDetails && isClassifier && ( - + {!isLoadingDetails && hasClassifierAudit && ( + )} - {!isLoadingDetails && !isClassifier && ( + {!isLoadingDetails && !hasClassifierAudit && ( rawRequest} getFormattedResponse={getFormattedResponse} logEntry={logEntry} /> From cc401c40419c71fe1fff24323d6165716f03cf3c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:16:01 -0700 Subject: [PATCH 057/126] fix(ui): skip organization fetches when the session is not premium --- .../organizations/useOrganizations.test.ts | 52 ++++++++++++++++--- .../hooks/organizations/useOrganizations.ts | 9 ++-- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.test.ts index 960afe7392c..5448b7182d7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.test.ts @@ -81,7 +81,7 @@ describe("useOrganizations", () => { userRole: "Admin", token: "test-token", userEmail: "test@example.com", - premiumUser: false, + premiumUser: true, disabledPersonalKeyCreation: null, showSSOBanner: false, }); @@ -181,7 +181,7 @@ describe("useOrganizations", () => { userRole: "Admin", token: null, userEmail: "test@example.com", - premiumUser: false, + premiumUser: true, disabledPersonalKeyCreation: null, showSSOBanner: false, }); @@ -197,6 +197,26 @@ describe("useOrganizations", () => { expect(organizationListCall).not.toHaveBeenCalled(); }); + it("does not call the organization API when the session is not premium", async () => { + mockUseAuthorized.mockReturnValue({ + accessToken: "test-access-token", + userId: "test-user-id", + userRole: "Admin", + token: "test-token", + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + + const { result } = renderHook(() => useOrganizations(), { wrapper }); + + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); + expect(result.current.isFetched).toBe(false); + expect(organizationListCall).not.toHaveBeenCalled(); + }); + it("should not execute query when userId is missing", async () => { // Mock missing userId mockUseAuthorized.mockReturnValue({ @@ -205,7 +225,7 @@ describe("useOrganizations", () => { userRole: "Admin", token: "test-token", userEmail: "test@example.com", - premiumUser: false, + premiumUser: true, disabledPersonalKeyCreation: null, showSSOBanner: false, }); @@ -229,7 +249,7 @@ describe("useOrganizations", () => { userRole: null, token: "test-token", userEmail: "test@example.com", - premiumUser: false, + premiumUser: true, disabledPersonalKeyCreation: null, showSSOBanner: false, }); @@ -253,7 +273,7 @@ describe("useOrganizations", () => { userRole: null, token: null, userEmail: "test@example.com", - premiumUser: false, + premiumUser: true, disabledPersonalKeyCreation: null, showSSOBanner: false, }); @@ -335,7 +355,7 @@ describe("useOrganization", () => { userRole: "Admin", token: "test-token", userEmail: "test@example.com", - premiumUser: false, + premiumUser: true, disabledPersonalKeyCreation: null, showSSOBanner: false, }); @@ -356,6 +376,26 @@ describe("useOrganization", () => { expect(result.current.isLoading).toBe(false); }); + it("does not call the organization info API when the session is not premium", () => { + mockUseAuthorized.mockReturnValue({ + accessToken: "test-access-token", + userId: "test-user-id", + userRole: "Admin", + token: "test-token", + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + + const { result } = renderHook(() => useOrganization("org-1"), { wrapper }); + + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); + expect(result.current.isFetched).toBe(false); + expect(organizationInfoCall).not.toHaveBeenCalled(); + }); + it("falls through to the detail API call when no cached list contains the organization", async () => { (organizationInfoCall as any).mockResolvedValue(mockOrganizations[0]); queryClient.setQueryData(organizationKeys.list({ filters: { org_id: "org-2" } }), [mockOrganizations[1]]); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts index 734c1986f8f..98053fdf038 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts @@ -11,9 +11,10 @@ export interface OrganizationListFilters { } export const useOrganizations = (filters?: OrganizationListFilters): UseQueryResult => { - const { accessToken, userId, userRole } = useAuthorized(); + const { accessToken, userId, userRole, premiumUser } = useAuthorized(); const orgId = filters?.org_id || null; const orgAlias = filters?.org_alias || null; + const hasSession = Boolean(accessToken && userId && userRole); return useQuery({ queryKey: organizationKeys.list( orgId || orgAlias @@ -21,16 +22,16 @@ export const useOrganizations = (filters?: OrganizationListFilters): UseQueryRes : {}, ), queryFn: async () => await organizationListCall(accessToken!, orgId, orgAlias), - enabled: Boolean(accessToken && userId && userRole), + enabled: hasSession && premiumUser === true, }); }; export const useOrganization = (organizationID?: string) => { const queryClient = useQueryClient(); - const { accessToken } = useAuthorized(); + const { accessToken, premiumUser } = useAuthorized(); return useQuery({ queryKey: organizationKeys.detail(organizationID!), - enabled: Boolean(accessToken && organizationID), + enabled: Boolean(accessToken && organizationID) && premiumUser === true, queryFn: async () => { if (!accessToken || !organizationID) { From b0d66a15b86a5e5d2f503d8c579b5e3a528ab3a0 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Thu, 10 Sep 2026 13:18:41 -0700 Subject: [PATCH 058/126] feat(ocr): add core foundation and Mistral adapter (#40530) * feat(ocr): add core foundation and transport primitives * fix(ocr): decline missing Mistral credentials * fix(rust): compile trace parity on Rust 1.98 * refactor(ocr): define native response capability * refactor(auth): generalize missing API key errors * refactor(core): keep URL helpers usage scoped * refactor(ocr): support native responses across adapters * refactor(ocr): preserve unmapped provider params * refactor(ocr): distinguish request preparation from payload transforms * refactor(ocr): trace payload transformation at codec boundary --- litellm-rust/Cargo.lock | 2 + litellm-rust/Cargo.toml | 1 + .../src/audio_transcription/hooks.rs | 2 +- .../crates/ai-gateway/src/ocr/hooks.rs | 2 +- .../ai-gateway/src/routes/messages/mod.rs | 3 +- .../crates/ai-gateway/src/trace_parity.rs | 8 +- litellm-rust/crates/core/Cargo.toml | 8 + litellm-rust/crates/core/src/constants.rs | 3 + litellm-rust/crates/core/src/error.rs | 107 +++++++++ litellm-rust/crates/core/src/http_utils.rs | 105 +++++++- litellm-rust/crates/core/src/lib.rs | 1 + .../crates/core/src/ocr/adapters/mistral.rs | 147 ++++++++++++ .../crates/core/src/ocr/adapters/mod.rs | 69 ++++++ litellm-rust/crates/core/src/ocr/client.rs | 75 ++++++ .../crates/core/src/ocr/codecs/mistral/mod.rs | 5 + .../src/ocr/codecs/mistral/transformation.rs | 102 ++++++++ .../core/src/ocr/codecs/mistral/types.rs | 53 ++++ .../crates/core/src/ocr/codecs/mod.rs | 1 + litellm-rust/crates/core/src/ocr/error.rs | 42 ++++ litellm-rust/crates/core/src/ocr/handler.rs | 72 ++++++ litellm-rust/crates/core/src/ocr/hooks.rs | 146 +++++++++++ litellm-rust/crates/core/src/ocr/mod.rs | 19 ++ litellm-rust/crates/core/src/ocr/prepare.rs | 142 +++++++++++ litellm-rust/crates/core/src/ocr/registry.rs | 53 ++++ .../crates/core/src/ocr/transformation.rs | 6 +- litellm-rust/crates/core/src/ocr/types.rs | 154 ++++++++++-- litellm-rust/crates/core/src/ocr/wire.rs | 154 ++++++++++++ .../providers/azure_ai/ocr/transformation.rs | 14 +- .../providers/mistral/ocr/transformation.rs | 11 +- .../providers/reducto/ocr/transformation.rs | 10 +- .../providers/vertex_ai/ocr/transformation.rs | 8 +- litellm-rust/crates/core/src/url_utils.rs | 95 ++++++++ litellm-rust/crates/core/tests/ocr.rs | 227 ++++++++++++++++++ litellm-rust/crates/core/tests/ocr/support.rs | 106 ++++++++ .../crates/python-bridge/src/errors.rs | 1 + 35 files changed, 1904 insertions(+), 50 deletions(-) create mode 100644 litellm-rust/crates/core/src/ocr/adapters/mistral.rs create mode 100644 litellm-rust/crates/core/src/ocr/adapters/mod.rs create mode 100644 litellm-rust/crates/core/src/ocr/client.rs create mode 100644 litellm-rust/crates/core/src/ocr/codecs/mistral/mod.rs create mode 100644 litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs create mode 100644 litellm-rust/crates/core/src/ocr/codecs/mistral/types.rs create mode 100644 litellm-rust/crates/core/src/ocr/codecs/mod.rs create mode 100644 litellm-rust/crates/core/src/ocr/error.rs create mode 100644 litellm-rust/crates/core/src/ocr/handler.rs create mode 100644 litellm-rust/crates/core/src/ocr/hooks.rs create mode 100644 litellm-rust/crates/core/src/ocr/prepare.rs create mode 100644 litellm-rust/crates/core/src/ocr/registry.rs create mode 100644 litellm-rust/crates/core/src/ocr/wire.rs create mode 100644 litellm-rust/crates/core/src/url_utils.rs create mode 100644 litellm-rust/crates/core/tests/ocr.rs create mode 100644 litellm-rust/crates/core/tests/ocr/support.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 43b9ec1aac2..7addc4e3e45 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -1453,11 +1453,13 @@ dependencies = [ "rstest", "serde", "serde_json", + "serde_path_to_error", "sha2 0.10.9", "thiserror 2.0.19", "tokio", "tracing", "tracing-subscriber", + "url", ] [[package]] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 82de7f40069..00fa6faaaef 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -39,6 +39,7 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } base64 = "0.22" +url = "2.5.8" [profile.release] opt-level = 3 diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs index dbe2d3a325b..098ce071efc 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs @@ -269,7 +269,7 @@ fn guardrail_error_to_core_error(error: GuardrailError) -> Error { fn core_error_kind(error: &Error) -> &'static str { match error { - Error::Auth(_) => "AuthError", + Error::Auth(_) | Error::MissingApiKey { .. } => "AuthError", Error::InvalidProvider(_) => "InvalidProvider", Error::InvalidRequest(_) => "InvalidRequest", Error::InvalidType { .. } => "InvalidType", diff --git a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs index ed41f1ff9e7..d1dd811f7ab 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs @@ -386,7 +386,7 @@ fn guardrail_error_to_core_error(error: GuardrailError) -> Error { fn core_error_kind(error: &Error) -> &'static str { match error { - Error::Auth(_) => "AuthError", + Error::Auth(_) | Error::MissingApiKey { .. } => "AuthError", Error::InvalidProvider(_) => "InvalidProvider", Error::InvalidRequest(_) => "InvalidRequest", Error::InvalidType { .. } => "InvalidType", diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs index bb9f3851a77..c22d05f5726 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs @@ -114,7 +114,8 @@ impl IntoResponse for MessagesRouteError { | Error::Connect(_) | Error::InvalidResponse(_) | Error::InvalidType { .. } - | Error::MissingField(_) => ( + | Error::MissingField(_) + | Error::MissingApiKey { .. } => ( StatusCode::BAD_GATEWAY, "messages provider request failed".to_string(), ), diff --git a/litellm-rust/crates/ai-gateway/src/trace_parity.rs b/litellm-rust/crates/ai-gateway/src/trace_parity.rs index 614852c541d..21123df3f1c 100644 --- a/litellm-rust/crates/ai-gateway/src/trace_parity.rs +++ b/litellm-rust/crates/ai-gateway/src/trace_parity.rs @@ -47,10 +47,10 @@ pub async fn messages_request( .header(CONTENT_TYPE, "application/json") .body(Body::from(body.to_string())) .map_err(|error| Error::InvalidRequest(error.to_string()))?; - let response = routes::app(state) - .oneshot(request) - .await - .map_err(|error| match error {})?; + let response = match routes::app(state).oneshot(request).await { + Ok(response) => response, + Err(error) => match error {}, + }; let status: StatusCode = response.status(); let bytes = to_bytes(response.into_body(), usize::MAX) .await diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index c0de7ff3977..b4ca88cf16f 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -4,6 +4,11 @@ version = "0.1.0" edition.workspace = true license.workspace = true repository.workspace = true +autotests = false + +[[test]] +name = "workspace_crate_allowlist" +path = "tests/workspace_crate_allowlist.rs" [dependencies] base64.workspace = true @@ -11,10 +16,13 @@ rand.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true +serde_path_to_error = "0.1" +tokio.workspace = true thiserror.workspace = true tracing.workspace = true tracing-subscriber = { workspace = true, optional = true } sha2.workspace = true +url.workspace = true aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true } aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"], optional = true } aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true } diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index fc81f4fa029..108d2a48e30 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -43,3 +43,6 @@ pub const EMPTY_TEXT_PLACEHOLDER: &str = "[System: Empty message content sanitised to satisfy protocol]"; pub const FUNCTION_TRACE_TARGET: &str = "litellm::function_trace"; +pub(crate) const OCR_HTTP_TIMEOUT_SECS: u64 = 600; +pub(crate) const OCR_CONNECT_TIMEOUT_SECS: u64 = 10; +pub(crate) const MISTRAL_OCR_API_BASE: &str = "https://api.mistral.ai/v1"; diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs index db3fa2ec704..0382314057f 100644 --- a/litellm-rust/crates/core/src/error.rs +++ b/litellm-rust/crates/core/src/error.rs @@ -17,6 +17,10 @@ pub enum Error { InvalidRequest(String), #[error("{0}")] Auth(String), + #[error( + "Missing {provider} API Key - A call is being made to {provider} but no key is set either in the environment variables or via params" + )] + MissingApiKey { provider: &'static str }, #[error("upstream request failed with status {status}: {body}")] Http { status: u16, body: String }, #[error("upstream network error: {0}")] @@ -36,6 +40,59 @@ pub enum Error { Unsupported(&'static str), } +#[derive(Clone, Debug, ThisError, PartialEq, Eq)] +pub enum TransportError { + #[error("upstream request failed with status {status}: {body}")] + Http { status: u16, body: String }, + #[error("upstream network error: {0}")] + Network(String), + #[error("could not reach the provider: {0}")] + Connect(String), +} + +impl TransportError { + pub fn from_reqwest_before_dispatch(error: reqwest::Error) -> Self { + let before_dispatch = !error.is_timeout() && (error.is_connect() || error.is_builder()); + let message = error.without_url().to_string(); + if before_dispatch { + Self::Connect(message) + } else { + Self::Network(message) + } + } +} + +impl From for TransportError { + fn from(error: reqwest::Error) -> Self { + Self::Network(error.without_url().to_string()) + } +} + +impl From for Error { + fn from(error: crate::ocr::error::OcrRequestError) -> Self { + match error { + crate::ocr::error::OcrRequestError::MissingField(field) => Self::MissingField(field), + error => Self::InvalidRequest(error.to_string()), + } + } +} + +impl From for Error { + fn from(error: crate::ocr::error::OcrResponseError) -> Self { + Self::InvalidResponse(error.to_string()) + } +} + +impl From for Error { + fn from(error: TransportError) -> Self { + match error { + TransportError::Http { status, body } => Self::Http { status, body }, + TransportError::Network(message) => Self::Network(message), + TransportError::Connect(message) => Self::Connect(message), + } + } +} + pub fn json_type_name(value: &serde_json::Value) -> &'static str { match value { serde_json::Value::Null => "null", @@ -46,3 +103,53 @@ pub fn json_type_name(value: &serde_json::Value) -> &'static str { serde_json::Value::Object(_) => "object", } } + +#[cfg(test)] +mod transport_tests { + use super::*; + + #[tokio::test] + async fn transport_errors_remove_urls_and_keep_dispatch_context() { + let error = reqwest::Client::builder() + .no_proxy() + .build() + .expect("client") + .get("http://localhost:invalid/private?api_key=secret") + .send() + .await + .expect_err("invalid port"); + let error = TransportError::from_reqwest_before_dispatch(error); + assert!(matches!(error, TransportError::Connect(_))); + assert!(!error.to_string().contains("secret")); + assert!(!error.to_string().contains("private")); + } + + #[tokio::test] + async fn request_timeout_is_not_safe_to_retry_as_a_connect_failure() { + use std::time::Duration; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let address = listener.local_addr().expect("address"); + let request = reqwest::Client::builder() + .no_proxy() + .build() + .expect("client") + .get(format!("http://{address}")) + .timeout(Duration::from_millis(200)) + .send(); + let (response, accepted) = tokio::join!( + request, + tokio::time::timeout(Duration::from_secs(2), listener.accept()) + ); + let _connection = accepted + .expect("accept deadline") + .expect("accepted connection"); + let error = response.expect_err("server does not respond"); + assert!(error.is_timeout()); + assert!(matches!( + TransportError::from_reqwest_before_dispatch(error), + TransportError::Network(_) + )); + } +} diff --git a/litellm-rust/crates/core/src/http_utils.rs b/litellm-rust/crates/core/src/http_utils.rs index cb472dd5a57..9299bb77ac8 100644 --- a/litellm-rust/crates/core/src/http_utils.rs +++ b/litellm-rust/crates/core/src/http_utils.rs @@ -1,10 +1,43 @@ -//! Header and upstream-body helpers shared by every route module. - use serde_json::{Map, Value}; use crate::constants::UPSTREAM_ERROR_BODY_MAX_CHARS; use crate::error::{Error, json_type_name}; +#[allow( + dead_code, + reason = "used by the OCR architecture in the next stacked PR" +)] +pub(crate) enum HeaderPolicy<'a> { + All, + Only(&'a [&'a str]), + Except(&'a [&'a str]), +} + +#[allow( + dead_code, + reason = "used by the OCR architecture in the next stacked PR" +)] +pub(crate) fn with_headers( + builder: reqwest::RequestBuilder, + headers: &[(String, String)], + policy: HeaderPolicy<'_>, +) -> reqwest::RequestBuilder { + headers + .iter() + .filter(|(name, _)| match policy { + HeaderPolicy::All => true, + HeaderPolicy::Only(names) => names + .iter() + .any(|allowed| name.eq_ignore_ascii_case(allowed)), + HeaderPolicy::Except(names) => !names + .iter() + .any(|excluded| name.eq_ignore_ascii_case(excluded)), + }) + .fold(builder, |builder, (name, value)| { + builder.header(name, value) + }) +} + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub async fn http_request( request: reqwest::RequestBuilder, @@ -12,8 +45,6 @@ pub async fn http_request( request.send().await } -/// Bound an upstream error body before it crosses a host boundary, so provider -/// bodies stay data-minimized. pub fn truncate_error_body(body: &str) -> String { if body.chars().count() <= UPSTREAM_ERROR_BODY_MAX_CHARS { return body.to_string(); @@ -61,11 +92,77 @@ pub fn has_bearer_auth(headers: &[(String, String)]) -> bool { }) } +#[allow( + dead_code, + reason = "used by the OCR architecture in the next stacked PR" +)] +pub(crate) fn deserialize_optional_param<'de, D, T>( + deserializer: D, +) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, + T: serde::Deserialize<'de>, +{ + as serde::Deserialize>::deserialize(deserializer).map(Some) +} + #[cfg(test)] mod tests { use super::*; use serde_json::json; + #[rstest::rstest] + #[case(HeaderPolicy::All, true, true)] + #[case(HeaderPolicy::Only(&["authorization"]), true, false)] + #[case(HeaderPolicy::Except(&["authorization"]), false, true)] + fn forwarding_policy_preserves_matching_headers_and_duplicates( + #[case] policy: HeaderPolicy<'_>, + #[case] auth: bool, + #[case] trace: bool, + ) { + let request = with_headers( + reqwest::Client::new().get("https://example.com"), + &[ + ("AuThOrIzAtIoN".into(), "Bearer token".into()), + ("X-Trace".into(), "first".into()), + ("x-trace".into(), "second".into()), + ], + policy, + ) + .build() + .unwrap(); + assert_eq!(request.headers().contains_key("authorization"), auth); + let traces: Vec<_> = request.headers().get_all("x-trace").iter().collect(); + if trace { + assert_eq!(traces, ["first", "second"]); + } else { + assert!(traces.is_empty()); + } + } + + #[test] + fn multipart_policy_leaves_content_headers_to_reqwest() { + let request = with_headers( + reqwest::Client::new() + .post("https://example.com") + .multipart(reqwest::multipart::Form::new().text("file", "abc")), + &[ + ("Content-Type".into(), "application/json".into()), + ("CONTENT-LENGTH".into(), "0".into()), + ], + HeaderPolicy::Except(&["content-type", "content-length"]), + ) + .build() + .unwrap(); + assert!( + request.headers()["content-type"] + .to_str() + .unwrap() + .starts_with("multipart/form-data; boundary=") + ); + assert_ne!(request.headers()["content-length"], "0"); + } + #[test] fn truncate_leaves_short_bodies_untouched() { assert_eq!(truncate_error_body("short"), "short"); diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index b93e084f57e..3a0896a4d5a 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -14,5 +14,6 @@ pub mod realtime; pub mod responses; pub mod router; pub mod routing_utils; +mod url_utils; pub use error::Error; diff --git a/litellm-rust/crates/core/src/ocr/adapters/mistral.rs b/litellm-rust/crates/core/src/ocr/adapters/mistral.rs new file mode 100644 index 00000000000..ea569ffb34f --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/adapters/mistral.rs @@ -0,0 +1,147 @@ +use super::OcrAdapter; +use crate::Error; +use crate::constants::MISTRAL_OCR_API_BASE; +use crate::ocr::OcrClient; +use crate::ocr::codecs::mistral::{self, MistralOcrParams, MistralOcrResponse}; +use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError}; +use crate::ocr::prepare::{ + _prepare_ocr_request, ParsedProviderParams, credential_env, transform_request_body, +}; +use crate::ocr::registry::OcrProvider; +use crate::ocr::types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection}; +use crate::url_utils::ApiUrl; + +const MISTRAL_API_KEY_ENV: &str = "MISTRAL_API_KEY"; + +#[derive(Clone, Debug)] +pub(crate) struct MistralAdapter; + +impl OcrAdapter for MistralAdapter { + type ProviderResponse = MistralOcrResponse; + const PROVIDER: OcrProvider = OcrProvider::Mistral; + + async fn prepare_request( + &self, + request: &LiteLLMOcrRequest, + client: &OcrClient, + ) -> Result { + let ParsedProviderParams { + known: params, + extra_params: _extra_params, + } = _prepare_ocr_request::(request)?; + let headers = validate_environment(&request.connection, &credential_env)?; + let url = get_complete_url(request.connection.api_base.as_deref())?; + let body = + mistral::transform_ocr_request(&request.model, request.document.clone(), ¶ms)?; + transform_request_body(client, request, &url, &headers, body, |_| Ok(())).await + } + + fn transform_ocr_response( + &self, + request: &LiteLLMOcrRequest, + response: Self::ProviderResponse, + ) -> Result { + mistral::transform_ocr_response(&request.model, response) + } +} + +pub(crate) fn get_complete_url(api_base: Option<&str>) -> Result { + let base = api_base + .map(str::trim) + .filter(|base| !base.is_empty()) + .unwrap_or(MISTRAL_OCR_API_BASE); + ApiUrl::parse(base) + .and_then(|url| url.complete_path(&["v1", "ocr"])) + .map(|url| url.into_string()) + .map_err(|_| { + OcrRequestError::RequestField { + path: "api_base".into(), + } + .into() + }) +} + +fn validate_environment( + connection: &OcrConnection, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> Result, OcrError> { + if crate::http_utils::has_header(&connection.extra_headers, "authorization") { + return Ok(connection.extra_headers.clone()); + } + let api_key = connection + .api_key + .as_deref() + .map(str::trim) + .filter(|key| !key.is_empty()) + .map(str::to_string) + .or_else(|| env_lookup(MISTRAL_API_KEY_ENV).filter(|key| !key.trim().is_empty())) + .ok_or(Error::MissingApiKey { + provider: "Mistral", + })?; + Ok( + std::iter::once(("Authorization".into(), format!("Bearer {api_key}"))) + .chain(connection.extra_headers.clone()) + .collect(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn complete_url_defaults_and_dedupes_v1() { + assert_eq!( + get_complete_url(None).unwrap(), + "https://api.mistral.ai/v1/ocr" + ); + assert_eq!( + get_complete_url(Some("https://example.com/v1?tenant=a")).unwrap(), + "https://example.com/v1/ocr?tenant=a" + ); + assert_eq!( + get_complete_url(Some("https://example.com/v1/ocr?tenant=a")).unwrap(), + "https://example.com/v1/ocr?tenant=a" + ); + } + + #[test] + fn environment_prefers_explicit_key_then_environment() { + let explicit = OcrConnection { + api_key: Some("explicit".into()), + ..OcrConnection::default() + }; + assert_eq!( + validate_environment(&explicit, &|_| Some("environment".into())).unwrap()[0], + ("Authorization".into(), "Bearer explicit".into()) + ); + + assert_eq!( + validate_environment(&OcrConnection::default(), &|_| Some("environment".into())) + .unwrap()[0], + ("Authorization".into(), "Bearer environment".into()) + ); + } + + #[test] + fn environment_preserves_forwarded_authorization() { + let connection = OcrConnection { + extra_headers: vec![("authorization".into(), "Bearer forwarded".into())], + ..OcrConnection::default() + }; + assert_eq!( + validate_environment(&connection, &|_| None).unwrap(), + connection.extra_headers + ); + } + + #[test] + fn environment_rejects_missing_key() { + assert!(matches!( + validate_environment(&OcrConnection::default(), &|_| None), + Err(OcrError::Public(Error::MissingApiKey { + provider: "Mistral" + })) + )); + } +} diff --git a/litellm-rust/crates/core/src/ocr/adapters/mod.rs b/litellm-rust/crates/core/src/ocr/adapters/mod.rs new file mode 100644 index 00000000000..7dfb08b4dc2 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/adapters/mod.rs @@ -0,0 +1,69 @@ +use std::future::Future; + +use serde::de::DeserializeOwned; + +use super::OcrClient; +use super::error::{OcrError, OcrResponseError}; +use super::registry::OcrProvider; +use super::types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrResponseFormat}; +use super::wire::DecodedOcrResponse; + +mod mistral; + +pub(crate) use mistral::MistralAdapter; + +/// Converts a complete LiteLLM OCR call to provider HTTP and normalizes its response. +pub(crate) trait OcrAdapter: Send + Sync + Sized + 'static { + /// Provider JSON schema; direct and Vertex Mistral share `MistralOcrResponse`. + type ProviderResponse: DeserializeOwned + Send; + + const PROVIDER: OcrProvider; + + /// Prepares the complete provider HTTP request. + /// `request` contains the model, document, connection, and unmapped caller options. + /// `client` supplies reusable provider and document HTTP clients. + /// Returns the complete HTTP request, whereas Python returns body data. + fn prepare_request( + &self, + request: &LiteLLMOcrRequest, + client: &OcrClient, + ) -> impl Future> + Send; + + /// Python: `transform_ocr_response`. + /// `request` supplies caller context, including the fallback model. + /// `response` is the decoded provider payload; the output is the shared LiteLLM schema. + fn transform_ocr_response( + &self, + request: &LiteLLMOcrRequest, + response: Self::ProviderResponse, + ) -> Result; + + /// Decodes provider HTTP; adapters may override this to poll asynchronous operations. + /// Python performs that polling inside `async_transform_ocr_response`. + /// `client` is reused for polling; `response` is the initial HTTP response. + /// `url` and `headers` describe the submitted call; `request` supplies limits and format. + fn read_response( + &self, + _client: &OcrClient, + response: reqwest::Response, + _url: &str, + _headers: &[(String, String)], + request: &LiteLLMOcrRequest, + ) -> impl Future, OcrError>> + Send + { + let retain_native = request + .response_format() + .map(|format| format == OcrResponseFormat::Native); + async move { super::client::read_json_response(response, retain_native?).await } + } +} + +macro_rules! for_each_ocr_adapter { + ($callback:ident) => { + $callback! { + Mistral, $crate::ocr::adapters::MistralAdapter, $crate::ocr::adapters::MistralAdapter, Mistral; + } + }; +} + +pub(crate) use for_each_ocr_adapter; diff --git a/litellm-rust/crates/core/src/ocr/client.rs b/litellm-rust/crates/core/src/ocr/client.rs new file mode 100644 index 00000000000..36bfc036bae --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/client.rs @@ -0,0 +1,75 @@ +use std::sync::OnceLock; +use std::time::Duration; + +use serde::de::DeserializeOwned; + +use super::error::OcrError; +use super::handler::perform_ocr_request; +use super::types::{LiteLLMOcrRequest, LiteLLMOcrResponse}; +use super::wire::{DecodedOcrResponse, decode_response}; +use crate::Error; +use crate::constants::OCR_CONNECT_TIMEOUT_SECS; +use crate::error::TransportError; + +#[derive(Clone)] +pub struct OcrClient { + provider_http: reqwest::Client, +} + +impl OcrClient { + pub fn new(provider_http: reqwest::Client) -> Result { + Ok(Self { provider_http }) + } + + #[tracing::instrument( + name = "ocr", + target = "litellm::function_trace", + level = "trace", + skip_all + )] + pub async fn perform(&self, request: LiteLLMOcrRequest) -> Result { + perform_ocr_request(self, request).await + } + + pub(crate) fn provider_http(&self) -> &reqwest::Client { + &self.provider_http + } + + #[cfg(test)] + pub(crate) fn for_test(provider_http: reqwest::Client) -> Self { + Self { provider_http } + } +} + +pub async fn ocr(request: LiteLLMOcrRequest) -> Result { + static CLIENT: OnceLock> = OnceLock::new(); + let client = CLIENT + .get_or_init(|| { + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(OCR_CONNECT_TIMEOUT_SECS)) + .build() + .map_err(TransportError::from) + .and_then(OcrClient::new) + }) + .clone()?; + client.perform(request).await +} + +pub async fn read_json_response( + response: reqwest::Response, + native: bool, +) -> Result, OcrError> { + let status = response.status(); + let bytes = response + .bytes() + .await + .map_err(crate::error::TransportError::from)?; + if !status.is_success() { + return Err(crate::error::TransportError::Http { + status: status.as_u16(), + body: crate::http_utils::truncate_error_body(&String::from_utf8_lossy(&bytes)), + } + .into()); + } + Ok(decode_response(&bytes, native)?) +} diff --git a/litellm-rust/crates/core/src/ocr/codecs/mistral/mod.rs b/litellm-rust/crates/core/src/ocr/codecs/mistral/mod.rs new file mode 100644 index 00000000000..eea4254779e --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/mistral/mod.rs @@ -0,0 +1,5 @@ +mod transformation; +mod types; + +pub(crate) use transformation::{transform_ocr_request, transform_ocr_response}; +pub(crate) use types::{MistralOcrParams, MistralOcrRequest, MistralOcrResponse}; diff --git a/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs b/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs new file mode 100644 index 00000000000..cd0a1dc6b17 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs @@ -0,0 +1,102 @@ +use super::{MistralOcrParams, MistralOcrRequest, MistralOcrResponse}; +use crate::ocr::error::{OcrRequestError, OcrResponseError}; +use crate::ocr::types::{LiteLLMOcrResponse, OcrDocument}; + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub(crate) fn transform_ocr_request( + model: &str, + document: OcrDocument, + params: &MistralOcrParams, +) -> Result { + Ok(MistralOcrRequest { + model: model.to_string(), + document, + params: params.clone(), + }) +} + +pub(crate) fn transform_ocr_response( + model: &str, + response: MistralOcrResponse, +) -> Result { + Ok(LiteLLMOcrResponse { + pages: response.pages, + model: response.model.unwrap_or_else(|| model.to_string()), + document_annotation: response.document_annotation, + usage_info: response.usage_info, + object: "ocr".to_string(), + extra_fields: response.extra_fields, + provider_native_response: None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + use serde_json::{Value, json}; + + #[rstest] + #[case("pages", json!([0, 2]))] + #[case("include_image_base64", json!(true))] + #[case("image_limit", json!(2))] + #[case("image_min_size", json!(100))] + #[case("bbox_annotation_format", json!({"type":"json_schema"}))] + #[case("document_annotation_format", json!({"type":"json_schema"}))] + #[case("document_annotation_prompt", json!("extract"))] + #[case("extract_header", json!(true))] + #[case("extract_footer", json!(false))] + #[case("table_format", json!("html"))] + #[case("confidence_scores_granularity", json!("word"))] + #[case("include_blocks", json!(true))] + #[case("id", json!("req-123"))] + fn request_mapping_matches_python(#[case] name: &str, #[case] value: Value) { + let params: MistralOcrParams = + serde_json::from_value(json!({name: value.clone()})).unwrap(); + let document: OcrDocument = serde_json::from_value( + json!({"type":"document_url","document_url":"https://example.com/a.pdf"}), + ) + .unwrap(); + let result = + serde_json::to_value(transform_ocr_request("model", document, ¶ms).unwrap()) + .unwrap(); + assert_eq!(result["model"], "model"); + assert_eq!(result[name], value); + } + + #[test] + fn request_mapping_filters_unknown_fields() { + let params: MistralOcrParams = serde_json::from_value(json!({"unknown": true})).unwrap(); + let document: OcrDocument = serde_json::from_value( + json!({"type":"document_url","document_url":"https://example.com/a.pdf"}), + ) + .unwrap(); + let result = + serde_json::to_value(transform_ocr_request("model", document, ¶ms).unwrap()) + .unwrap(); + assert!(result.get("unknown").is_none()); + } + + #[test] + fn response_preserves_provider_fields() { + let response: MistralOcrResponse = serde_json::from_value(json!({ + "pages":[{"index":0,"markdown":"hello","header":"head","confidence_scores":{"mean":0.99}}], + "model":"returned-model", + "usage_info":{"pages_processed":1,"future_counter":5}, + "future_response_field":"kept" + })) + .unwrap(); + let result = transform_ocr_response("model", response) + .unwrap() + .into_json(); + assert_eq!(result["pages"][0]["header"], "head"); + assert_eq!(result["usage_info"]["future_counter"], 5); + assert_eq!(result["future_response_field"], "kept"); + assert_eq!(result["model"], "returned-model"); + } + + #[test] + fn response_rejects_null_pages() { + assert!(serde_json::from_value::(json!({"pages":null})).is_err()); + } +} diff --git a/litellm-rust/crates/core/src/ocr/codecs/mistral/types.rs b/litellm-rust/crates/core/src/ocr/codecs/mistral/types.rs new file mode 100644 index 00000000000..0e601cd8319 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/mistral/types.rs @@ -0,0 +1,53 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use crate::ocr::types::OcrDocument; + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub(crate) struct MistralOcrParams { + #[serde(skip_serializing_if = "Option::is_none")] + pub pages: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub include_image_base64: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub image_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub image_min_size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub bbox_annotation_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub document_annotation_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub document_annotation_prompt: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub extract_header: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub extract_footer: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub table_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub confidence_scores_granularity: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub include_blocks: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct MistralOcrRequest { + pub model: String, + pub document: OcrDocument, + #[serde(flatten)] + pub params: MistralOcrParams, +} + +#[derive(Clone, Debug, Default, Deserialize)] +pub(crate) struct MistralOcrResponse { + #[serde(default)] + pub pages: Vec, + pub model: Option, + pub document_annotation: Option, + pub usage_info: Option, + #[serde(flatten)] + pub extra_fields: Map, +} diff --git a/litellm-rust/crates/core/src/ocr/codecs/mod.rs b/litellm-rust/crates/core/src/ocr/codecs/mod.rs new file mode 100644 index 00000000000..170ef5f68a7 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/mod.rs @@ -0,0 +1 @@ +pub(crate) mod mistral; diff --git a/litellm-rust/crates/core/src/ocr/error.rs b/litellm-rust/crates/core/src/ocr/error.rs new file mode 100644 index 00000000000..50793aad566 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/error.rs @@ -0,0 +1,42 @@ +use thiserror::Error; + +use crate::error::TransportError; + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum OcrRequestError { + #[error("Invalid `req_format`. Expected 'native' or 'litellm'.")] + RequestFormat, + #[error("invalid OCR request field: {path}")] + RequestField { path: String }, + #[error("missing required field: {0}")] + MissingField(&'static str), +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum OcrResponseError { + #[error("invalid OCR response field: {path}")] + ResponseField { path: String }, +} + +#[derive(Debug, Error)] +pub enum OcrError { + #[error("{0}")] + Request(#[from] OcrRequestError), + #[error("{0}")] + Response(#[from] OcrResponseError), + #[error("{0}")] + Transport(#[from] TransportError), + #[error("{0}")] + Public(#[from] crate::Error), +} + +impl From for crate::Error { + fn from(error: OcrError) -> Self { + match error { + OcrError::Request(error) => error.into(), + OcrError::Response(error) => error.into(), + OcrError::Transport(error) => error.into(), + OcrError::Public(error) => error, + } + } +} diff --git a/litellm-rust/crates/core/src/ocr/handler.rs b/litellm-rust/crates/core/src/ocr/handler.rs new file mode 100644 index 00000000000..0b04319d966 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/handler.rs @@ -0,0 +1,72 @@ +use super::OcrClient; +use super::adapters::OcrAdapter; +use super::hooks::OcrLifecycleHooks; +use super::registry::OcrAdapterKind; +use super::types::{LiteLLMOcrRequest, LiteLLMOcrResponse}; +use crate::Error; +use crate::call_lifecycle::{CallLifecycle, CallLifecycleContext}; + +pub(crate) async fn perform_ocr_request( + client: &OcrClient, + request: LiteLLMOcrRequest, +) -> Result { + let context = CallLifecycleContext::new( + "ocr", + request.model.clone(), + request.adapter.provider().as_str(), + request + .litellm_call_id + .clone() + .unwrap_or_else(|| format!("ocr-{:032x}", rand::random::())), + ); + let hooks = OcrLifecycleHooks { + hooks: request.hooks.clone(), + provider_name: context.custom_llm_provider.clone(), + }; + CallLifecycle::default().run(context, request, &hooks, |request| async move { + macro_rules! execute_selected_adapter { + ($( $variant:ident, $adapter:ty, $instance:expr, $provider:ident; )+) => { + match request.adapter { + $( OcrAdapterKind::$variant => execute_ocr_provider_call(client, &$instance, request).await, )+ + } + }; + } + super::adapters::for_each_ocr_adapter!(execute_selected_adapter) + }).await +} + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +async fn execute_ocr_provider_call( + client: &OcrClient, + adapter: &A, + request: LiteLLMOcrRequest, +) -> Result { + let provider_request = adapter.prepare_request(&request, client).await?; + let url = provider_request.url().to_string(); + let headers = provider_request + .headers() + .iter() + .map(|(name, value)| { + value + .to_str() + .map(|value| (name.to_string(), value.to_string())) + .map_err(|_| super::error::OcrRequestError::RequestField { + path: "headers".into(), + }) + }) + .collect::, _>>()?; + let response = crate::http_utils::http_request(reqwest::RequestBuilder::from_parts( + client.provider_http().clone(), + provider_request, + )) + .await + .map_err(crate::error::TransportError::from)?; + let decoded = adapter + .read_response(client, response, &url, &headers, &request) + .await?; + let response = adapter.transform_ocr_response(&request, decoded.data)?; + Ok(LiteLLMOcrResponse { + provider_native_response: decoded.native, + ..response + }) +} diff --git a/litellm-rust/crates/core/src/ocr/hooks.rs b/litellm-rust/crates/core/src/ocr/hooks.rs new file mode 100644 index 00000000000..7dd3c6bf8b2 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/hooks.rs @@ -0,0 +1,146 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use super::types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrDocument}; +use crate::Error; +use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; +use serde::Serialize; +use serde_json::Value; + +pub type OcrHookFuture<'a, T> = Pin> + Send + 'a>>; +pub type OcrLogFuture<'a> = Pin + Send + 'a>>; + +#[derive(Clone, Debug, Serialize)] +pub struct OcrPreCallRequest { + pub model: String, + pub custom_llm_provider: String, + pub document: OcrDocument, + pub optional_params: Value, +} + +#[derive(Clone, Debug, Serialize)] +pub struct OcrDuringCallRequest { + pub model: String, + pub custom_llm_provider: String, + pub url: String, + pub body: Value, +} + +pub trait OcrHooks: Send + Sync { + fn has_guardrails(&self) -> bool { + false + } + fn pre_call(&self, request: OcrPreCallRequest) -> OcrHookFuture<'_, OcrPreCallRequest> { + Box::pin(async move { Ok(request) }) + } + fn during_call( + &self, + request: OcrDuringCallRequest, + ) -> OcrHookFuture<'_, OcrDuringCallRequest> { + Box::pin(async move { Ok(request) }) + } + fn success<'a>( + &'a self, + _context: &'a CallLifecycleContext, + _response: &'a LiteLLMOcrResponse, + _timing: &'a CallLifecycleTiming, + ) -> OcrLogFuture<'a> { + Box::pin(async {}) + } + fn failure<'a>( + &'a self, + _context: &'a CallLifecycleContext, + _error: &'a Error, + _timing: &'a CallLifecycleTiming, + ) -> OcrLogFuture<'a> { + Box::pin(async {}) + } +} + +pub struct NoopOcrHooks; +impl OcrHooks for NoopOcrHooks {} + +pub(crate) struct OcrLifecycleHooks { + pub hooks: Arc, + pub provider_name: String, +} + +impl CallLifecycleHooks + for OcrLifecycleHooks +{ + type PreCallFuture<'a> = OcrHookFuture<'a, LiteLLMOcrRequest>; + type DuringCallFuture<'a> = OcrHookFuture<'a, LiteLLMOcrRequest>; + type SuccessFuture<'a> = OcrLogFuture<'a>; + type FailureFuture<'a> = OcrLogFuture<'a>; + + fn async_pre_call_hook<'a>( + &'a self, + _context: &'a CallLifecycleContext, + request: LiteLLMOcrRequest, + ) -> Self::PreCallFuture<'a> { + Box::pin(async move { + if !self.hooks.has_guardrails() { + return Ok(request); + } + let changed = self + .hooks + .pre_call(OcrPreCallRequest { + model: request.model.clone(), + custom_llm_provider: self.provider_name.clone(), + document: request.document, + optional_params: Value::Object(request.optional_params), + }) + .await?; + let Value::Object(optional_params) = changed.optional_params else { + return Err(super::error::OcrRequestError::RequestField { + path: "guardrail.optional_params".into(), + } + .into()); + }; + Ok(LiteLLMOcrRequest { + document: changed.document, + optional_params, + ..request + }) + }) + } + + fn async_during_call_hook<'a>( + &'a self, + _context: &'a CallLifecycleContext, + request: LiteLLMOcrRequest, + ) -> Self::DuringCallFuture<'a> { + Box::pin(async move { Ok(request) }) + } + + #[tracing::instrument( + name = "success_callback", + target = "litellm::function_trace", + level = "trace", + skip_all + )] + fn async_log_success_event<'a>( + &'a self, + context: &'a CallLifecycleContext, + response: &'a LiteLLMOcrResponse, + timing: &'a CallLifecycleTiming, + ) -> Self::SuccessFuture<'a> { + self.hooks.success(context, response, timing) + } + + #[tracing::instrument( + name = "failure_callback", + target = "litellm::function_trace", + level = "trace", + skip_all + )] + fn async_log_failure_event<'a>( + &'a self, + context: &'a CallLifecycleContext, + error: &'a Error, + timing: &'a CallLifecycleTiming, + ) -> Self::FailureFuture<'a> { + self.hooks.failure(context, error, timing) + } +} diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs index ec2fbb969a6..78c86aeaa0e 100644 --- a/litellm-rust/crates/core/src/ocr/mod.rs +++ b/litellm-rust/crates/core/src/ocr/mod.rs @@ -1,2 +1,21 @@ +mod adapters; +pub mod client; +mod codecs; +pub mod error; +mod handler; +pub mod hooks; +mod prepare; +mod registry; pub mod transformation; pub mod types; +pub mod wire; + +pub use client::{OcrClient, ocr}; +pub use types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection, OcrDocument}; + +#[cfg(test)] +#[path = "../../tests/ocr/support.rs"] +pub(crate) mod test_support; +#[cfg(test)] +#[path = "../../tests/ocr.rs"] +pub(crate) mod tests; diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs new file mode 100644 index 00000000000..cff40b7de50 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -0,0 +1,142 @@ +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use serde_json::{Map, Value}; + +use super::OcrClient; +use super::error::{OcrError, OcrRequestError}; +use super::hooks::OcrDuringCallRequest; +use super::types::LiteLLMOcrRequest; + +#[derive(Debug, Deserialize)] +pub(crate) struct ParsedProviderParams { + #[serde(flatten)] + pub known: T, + #[serde(default, flatten)] + pub extra_params: Map, +} + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub(crate) fn _prepare_ocr_request( + request: &LiteLLMOcrRequest, +) -> Result, OcrRequestError> { + super::wire::decode_request_value( + Value::Object(request.optional_params.clone()), + "optional_params", + ) +} + +pub(crate) async fn transform_request_body( + client: &OcrClient, + request: &LiteLLMOcrRequest, + url: &str, + headers: &[(String, String)], + body: B, + validate: impl FnOnce(&B) -> Result<(), OcrRequestError>, +) -> Result +where + B: Serialize + DeserializeOwned, +{ + let body = if request.hooks.has_guardrails() { + let changed = request + .hooks + .during_call(OcrDuringCallRequest { + model: request.model.clone(), + custom_llm_provider: request.adapter.provider().as_str().into(), + url: url.into(), + body: serde_json::to_value(body).map_err(|_| OcrRequestError::RequestField { + path: "body".into(), + })?, + }) + .await?; + let body = OcrWireBody::::decode(changed.body)?; + validate(&body.body)?; + body + } else { + OcrWireBody { + body, + extra: Map::new(), + } + }; + build_http_request(client, request, url, headers, &body) +} + +pub(crate) fn build_http_request( + client: &OcrClient, + request: &LiteLLMOcrRequest, + url: &str, + headers: &[(String, String)], + body: &B, +) -> Result { + let builder = client + .provider_http() + .post(url) + .json(body) + .timeout(request.connection.timeout); + crate::http_utils::with_headers(builder, headers, crate::http_utils::HeaderPolicy::All) + .build() + .map_err(crate::error::TransportError::from) + .map_err(OcrError::from) +} + +#[derive(Serialize)] +struct OcrWireBody { + #[serde(flatten)] + body: B, + #[serde(flatten)] + extra: Map, +} + +impl OcrWireBody { + fn decode(value: Value) -> Result { + let body: B = super::wire::decode_request_value(value.clone(), "guardrail.body")?; + let Value::Object(fields) = value else { + return Err(OcrRequestError::RequestField { + path: "guardrail.body".into(), + }); + }; + let known = serde_json::to_value(&body).map_err(|_| OcrRequestError::RequestField { + path: "guardrail.body".into(), + })?; + let extra = fields + .into_iter() + .filter(|(key, _)| known.get(key).is_none()) + .collect(); + Ok(Self { body, extra }) + } +} + +pub(crate) fn credential_env(name: &str) -> Option { + std::env::var(name).ok() +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[derive(Debug, Deserialize, PartialEq)] + struct KnownParams { + pages: Option>, + } + + #[test] + fn parsed_provider_params_separates_known_and_extra_params() { + let parsed: ParsedProviderParams = super::super::wire::decode_request_value( + json!({ + "pages": [0, 2], + "future_ocr_option": true, + "extra_body": {"provider_option": "value"} + }), + "optional_params", + ) + .unwrap(); + + assert_eq!(parsed.known.pages, Some(vec![0, 2])); + assert_eq!(parsed.extra_params["future_ocr_option"], true); + assert_eq!( + parsed.extra_params["extra_body"], + json!({"provider_option": "value"}) + ); + assert_eq!(parsed.extra_params.len(), 2); + } +} diff --git a/litellm-rust/crates/core/src/ocr/registry.rs b/litellm-rust/crates/core/src/ocr/registry.rs new file mode 100644 index 00000000000..9bae8153ee8 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/registry.rs @@ -0,0 +1,53 @@ +use super::adapters::OcrAdapter; +use crate::Error; +use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; + +macro_rules! define_adapter_types { + ($( $variant:ident, $adapter:ty, $instance:expr, $provider:ident; )+) => { + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(crate) enum OcrAdapterKind { + $( $variant, )+ + } + + impl OcrAdapterKind { + pub(crate) const fn provider(self) -> OcrProvider { + match self { + $( Self::$variant => <$adapter>::PROVIDER, )+ + } + } + } + }; +} + +super::adapters::for_each_ocr_adapter!(define_adapter_types); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum OcrProvider { + Mistral, +} + +impl OcrProvider { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Mistral => "mistral", + } + } +} + +pub(crate) fn resolve_wire_adapter( + model: &str, + custom_llm_provider: Option<&str>, +) -> Result<(String, OcrAdapterKind), Error> { + let provider = + get_custom_llm_provider(model, custom_llm_provider).unwrap_or(CustomLlmProvider { + model, + custom_llm_provider: OcrProvider::Mistral.as_str(), + }); + let typed_provider = match provider.custom_llm_provider { + "mistral" => OcrProvider::Mistral, + value => return Err(Error::InvalidProvider(value.to_string())), + }; + match typed_provider { + OcrProvider::Mistral => Ok((provider.model.to_string(), OcrAdapterKind::Mistral)), + } +} diff --git a/litellm-rust/crates/core/src/ocr/transformation.rs b/litellm-rust/crates/core/src/ocr/transformation.rs index 62299faf9ed..ac4f10bf15b 100644 --- a/litellm-rust/crates/core/src/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/transformation.rs @@ -1,7 +1,7 @@ use crate::Error; use serde_json::{Map, Value}; -use super::types::{OcrRequestData, OcrResponseData}; +use super::types::{LiteLLMOcrResponse, OcrRequestData}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum OcrAuthStrategy { @@ -49,14 +49,14 @@ pub trait OcrProviderConfig: Sync { &self, model: &str, response_json: Value, - ) -> Result; + ) -> Result; fn transform_ocr_response_with_params( &self, model: &str, response_json: Value, _optional_params: &Map, - ) -> Result { + ) -> Result { self.transform_ocr_response(model, response_json) } diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/core/src/ocr/types.rs index 71cdb232a87..02b5330188c 100644 --- a/litellm-rust/crates/core/src/ocr/types.rs +++ b/litellm-rust/crates/core/src/ocr/types.rs @@ -1,6 +1,14 @@ +use std::sync::Arc; +use std::time::Duration; + use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +use super::hooks::{NoopOcrHooks, OcrHooks}; +use super::registry::{OcrAdapterKind, resolve_wire_adapter}; +use crate::Error; +use crate::constants::OCR_HTTP_TIMEOUT_SECS; + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct OcrRequestData { pub data: Value, @@ -8,31 +16,145 @@ pub struct OcrRequestData { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct OcrResponseData { +#[serde(tag = "type")] +pub enum OcrDocument { + #[serde(rename = "document_url")] + DocumentUrl { + document_url: String, + #[serde(flatten)] + extra_fields: Map, + }, + #[serde(rename = "image_url")] + ImageUrl { + image_url: String, + #[serde(flatten)] + extra_fields: Map, + }, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum OcrResponseFormat { + #[default] + Litellm, + Native, +} + +#[derive(Clone)] +pub struct OcrConnection { + pub api_key: Option, + pub api_base: Option, + pub extra_headers: Vec<(String, String)>, + pub timeout: Duration, +} + +impl Default for OcrConnection { + fn default() -> Self { + Self { + api_key: None, + api_base: None, + extra_headers: Vec::new(), + timeout: Duration::from_secs(OCR_HTTP_TIMEOUT_SECS), + } + } +} + +pub struct LiteLLMOcrRequest { + pub model: String, + pub document: OcrDocument, + pub connection: OcrConnection, + pub hooks: Arc, + pub litellm_call_id: Option, + pub optional_params: Map, + pub(crate) adapter: OcrAdapterKind, +} + +impl LiteLLMOcrRequest { + pub fn new( + model: String, + document: OcrDocument, + custom_llm_provider: Option<&str>, + optional_params: Map, + ) -> Result { + let (model, adapter_kind) = resolve_wire_adapter(&model, custom_llm_provider)?; + + Ok(Self { + model, + document, + connection: OcrConnection::default(), + hooks: Arc::new(NoopOcrHooks), + litellm_call_id: None, + optional_params, + adapter: adapter_kind, + }) + } + + pub(crate) fn response_format( + &self, + ) -> Result { + self.optional_params + .get("req_format") + .map(|value| { + serde_json::from_value(value.clone()) + .map_err(|_| super::error::OcrRequestError::RequestFormat) + }) + .transpose() + .map(|format| format.unwrap_or_default()) + } + + pub fn with_host_hooks( + self, + hooks: Arc, + litellm_call_id: Option, + ) -> Self { + Self { + hooks, + litellm_call_id, + ..self + } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct LiteLLMOcrResponse { pub pages: Vec, pub model: String, pub document_annotation: Option, pub usage_info: Option, pub object: String, + #[serde(flatten)] pub extra_fields: Map, + #[serde(skip_serializing_if = "Option::is_none")] pub provider_native_response: Option, } -impl OcrResponseData { +impl LiteLLMOcrResponse { pub fn into_json(self) -> Value { - let mut response = serde_json::json!({ - "pages": self.pages, - "model": self.model, - "document_annotation": self.document_annotation, - "usage_info": self.usage_info, - "object": self.object, - }); - if let Value::Object(object) = &mut response { - object.extend(self.extra_fields); - if let Some(native_response) = self.provider_native_response { - object.insert("provider_native_response".to_string(), native_response); - } - } - response + serde_json::to_value(self).expect("OCR response fields are JSON-compatible") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn response_serialization_flattens_extra_fields_and_omits_absent_native_response() { + let response = LiteLLMOcrResponse { + pages: vec![], + model: "model".into(), + document_annotation: None, + usage_info: None, + object: "ocr".into(), + extra_fields: json!({"provider_field":"kept"}) + .as_object() + .unwrap() + .clone(), + provider_native_response: None, + }; + let serialized = response.into_json(); + assert_eq!(serialized["provider_field"], "kept"); + assert!(serialized.get("provider_native_response").is_none()); } } diff --git a/litellm-rust/crates/core/src/ocr/wire.rs b/litellm-rust/crates/core/src/ocr/wire.rs new file mode 100644 index 00000000000..f37c06a01ac --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/wire.rs @@ -0,0 +1,154 @@ +use crate::ocr::error::OcrRequestError; +use crate::ocr::error::OcrResponseError; +use std::time::Duration; + +use super::hooks::{OcrDuringCallRequest, OcrPreCallRequest}; +use super::types::{LiteLLMOcrRequest, OcrConnection, OcrDocument}; +use crate::Error; +use serde::{ + Deserialize, + de::{DeserializeOwned, IntoDeserializer}, +}; +use serde_json::{Map, Value}; + +#[derive(Debug)] +pub struct DecodedOcrResponse { + pub data: T, + pub native: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OcrWireRequest { + pub model: String, + pub document: Value, + pub api_key: Option, + pub api_base: Option, + pub custom_llm_provider: Option, + pub extra_headers: Option>, + #[serde(default)] + pub optional_params: Map, + pub timeout_seconds: Option, +} + +pub fn is_supported_request(model: &str, custom_llm_provider: Option<&str>) -> bool { + super::registry::resolve_wire_adapter(model, custom_llm_provider).is_ok() +} + +pub fn decode_request(wire: OcrWireRequest) -> Result { + let document = decode_request_value(wire.document, "document")?; + let headers = wire + .extra_headers + .unwrap_or_default() + .into_iter() + .map(|(name, value)| { + let value = value + .as_str() + .ok_or_else(|| OcrRequestError::RequestField { + path: format!("extra_headers.{name}"), + })?; + Ok((name, value.to_string())) + }) + .collect::, OcrRequestError>>()?; + let timeout = wire + .timeout_seconds + .map(|seconds| { + Duration::try_from_secs_f64(seconds).map_err(|_| OcrRequestError::RequestField { + path: "timeout_seconds".into(), + }) + }) + .transpose()?; + let defaults = OcrConnection::default(); + let request = LiteLLMOcrRequest::new( + wire.model, + document, + wire.custom_llm_provider.as_deref(), + wire.optional_params, + )?; + let connection = OcrConnection { + api_key: nonblank(wire.api_key), + api_base: nonblank(wire.api_base), + extra_headers: headers, + timeout: timeout.unwrap_or(defaults.timeout), + }; + Ok(LiteLLMOcrRequest { + connection, + ..request + }) +} + +fn nonblank(value: Option) -> Option { + value + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} +pub fn decode_request_value( + value: Value, + prefix: &str, +) -> Result { + serde_path_to_error::deserialize(value.into_deserializer()).map_err(|error| { + OcrRequestError::RequestField { + path: format!("{prefix}.{}", error.path()), + } + }) +} + +pub fn decode_response( + bytes: &[u8], + native: bool, +) -> Result, OcrResponseError> { + let mut deserializer = serde_json::Deserializer::from_slice(bytes); + let data = serde_path_to_error::deserialize(&mut deserializer).map_err(|error| { + OcrResponseError::ResponseField { + path: error.path().to_string(), + } + })?; + deserializer + .end() + .map_err(|_| OcrResponseError::ResponseField { + path: "response".into(), + })?; + let native = if native { + Some( + serde_json::from_slice(bytes).map_err(|_| OcrResponseError::ResponseField { + path: "response".into(), + })?, + ) + } else { + None + }; + Ok(DecodedOcrResponse { data, native }) +} + +pub fn decode_pre_call_result( + original: OcrPreCallRequest, + value: Value, +) -> Result { + #[derive(Deserialize)] + struct Changed { + document: OcrDocument, + #[serde(default)] + optional_params: Map, + } + let changed: Changed = decode_request_value(value, "guardrail")?; + Ok(OcrPreCallRequest { + document: changed.document, + optional_params: Value::Object(changed.optional_params), + ..original + }) +} + +pub fn decode_during_call_result( + original: OcrDuringCallRequest, + value: Value, +) -> Result { + #[derive(Deserialize)] + struct Changed { + body: Value, + } + let changed: Changed = decode_request_value(value, "guardrail")?; + Ok(OcrDuringCallRequest { + body: changed.body, + ..original + }) +} diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs index d15c032f0bc..2af25a5639a 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs @@ -2,7 +2,7 @@ use std::collections::BTreeSet; use crate::error::{Error, json_type_name}; use crate::ocr::transformation::{OcrAuthStrategy, OcrProviderConfig, OcrResponseHandling}; -use crate::ocr::types::{OcrRequestData, OcrResponseData}; +use crate::ocr::types::{LiteLLMOcrResponse, OcrRequestData}; use serde_json::{Map, Value, json}; use crate::providers::mistral::ocr::transformation::MISTRAL_OCR_CONFIG; @@ -440,7 +440,7 @@ fn transform_document_intelligence_response( model: &str, response_json: Value, preserve_native_response: bool, -) -> Result { +) -> Result { let response = response_json .as_object() .ok_or_else(|| Error::InvalidType { @@ -488,7 +488,7 @@ fn transform_document_intelligence_response( }) .collect(); - Ok(OcrResponseData { + Ok(LiteLLMOcrResponse { usage_info: Some(json!({ "pages_processed": pages.len(), "doc_size_bytes": null, @@ -521,7 +521,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { &self, model: &str, response_json: Value, - ) -> Result { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -599,7 +599,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { &self, model: &str, response_json: Value, - ) -> Result { + ) -> Result { transform_document_intelligence_response(model, response_json, false) } @@ -608,7 +608,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { model: &str, response_json: Value, optional_params: &Map, - ) -> Result { + ) -> Result { transform_document_intelligence_response( model, response_json, @@ -718,7 +718,7 @@ mod tests { }) } - fn assert_native_fields_preserved(response: &OcrResponseData, operation: &Value) { + fn assert_native_fields_preserved(response: &LiteLLMOcrResponse, operation: &Value) { let analyze_result = &operation["analyzeResult"]; assert_eq!(response.extra_fields["content"], analyze_result["content"]); diff --git a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs index 11e8fe7db18..044fc587c22 100644 --- a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs @@ -1,6 +1,6 @@ use crate::error::{Error, json_type_name}; use crate::ocr::transformation::OcrProviderConfig; -use crate::ocr::types::{OcrRequestData, OcrResponseData}; +use crate::ocr::types::{LiteLLMOcrResponse, OcrRequestData}; use serde_json::{Map, Value}; const SUPPORTED_OCR_PARAMS: &[&str] = &[ @@ -107,7 +107,7 @@ impl OcrProviderConfig for MistralOcrConfig { &self, model: &str, response_json: Value, - ) -> Result { + ) -> Result { let response_object = response_json .as_object() .ok_or_else(|| Error::InvalidType { @@ -128,7 +128,7 @@ impl OcrProviderConfig for MistralOcrConfig { let document_annotation = response_object.get("document_annotation").cloned(); let usage_info = response_object.get("usage_info").cloned(); - Ok(OcrResponseData { + Ok(LiteLLMOcrResponse { pages, model, document_annotation, @@ -178,7 +178,10 @@ pub fn transform_ocr_request( } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub fn transform_ocr_response(model: &str, response_json: Value) -> Result { +pub fn transform_ocr_response( + model: &str, + response_json: Value, +) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } diff --git a/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs index b8507541e18..f025887846f 100644 --- a/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs @@ -6,7 +6,7 @@ use serde_json::{Map, Value, json}; use crate::error::{Error, json_type_name}; use crate::ocr::transformation::OcrProviderConfig; -use crate::ocr::types::{OcrRequestData, OcrResponseData}; +use crate::ocr::types::{LiteLLMOcrResponse, OcrRequestData}; pub const REDUCTO_API_BASE: &str = "https://platform.reducto.ai"; pub const REDUCTO_API_KEY_ENV: &str = "REDUCTO_API_KEY"; @@ -283,7 +283,7 @@ fn build_pages(result: &Map) -> Vec { pub fn transform_reducto_response( model: &str, response_json: Value, -) -> Result { +) -> Result { let response = response_json .as_object() .ok_or_else(|| Error::InvalidType { @@ -311,7 +311,7 @@ pub fn transform_reducto_response( "credits": usage.get("credits").cloned().unwrap_or(Value::Null), })); - Ok(OcrResponseData { + Ok(LiteLLMOcrResponse { pages: build_pages(result), model: model.to_string(), document_annotation: None, @@ -341,7 +341,7 @@ impl OcrProviderConfig for ReductoParseV3Config { &self, model: &str, response_json: Value, - ) -> Result { + ) -> Result { transform_reducto_response(model, response_json) } @@ -383,7 +383,7 @@ impl OcrProviderConfig for ReductoParseLegacyConfig { &self, model: &str, response_json: Value, - ) -> Result { + ) -> Result { transform_reducto_response(model, response_json) } diff --git a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs index c324de8cb45..b0e5a278d0b 100644 --- a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs @@ -1,6 +1,6 @@ use crate::error::{Error, json_type_name}; use crate::ocr::transformation::OcrProviderConfig; -use crate::ocr::types::{OcrRequestData, OcrResponseData}; +use crate::ocr::types::{LiteLLMOcrResponse, OcrRequestData}; use serde_json::{Map, Value, json}; use crate::providers::mistral::ocr::transformation::MISTRAL_OCR_CONFIG; @@ -226,7 +226,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { &self, model: &str, response_json: Value, - ) -> Result { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -301,7 +301,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { &self, model: &str, response_json: Value, - ) -> Result { + ) -> Result { let response = response_json .as_object() .ok_or_else(|| Error::InvalidType { @@ -339,7 +339,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { .get("usage_info") .cloned() .or_else(|| response.get("usage").cloned()); - Ok(OcrResponseData { + Ok(LiteLLMOcrResponse { pages, model: object .get("model") diff --git a/litellm-rust/crates/core/src/url_utils.rs b/litellm-rust/crates/core/src/url_utils.rs new file mode 100644 index 00000000000..982dca0dbe3 --- /dev/null +++ b/litellm-rust/crates/core/src/url_utils.rs @@ -0,0 +1,95 @@ +use std::marker::PhantomData; + +use thiserror::Error; +use url::Url; + +#[derive(Debug, Error)] +pub(crate) enum ApiUrlError { + #[error("invalid URL: {0}")] + Parse(#[from] url::ParseError), + #[error("URL cannot be used as a base")] + CannotBeBase, +} + +pub(crate) struct Base; +pub(crate) struct Complete; + +pub(crate) struct ApiUrl { + url: Url, + state: PhantomData, +} + +impl ApiUrl { + pub(crate) fn parse(value: &str) -> Result { + Ok(Self { + url: Url::parse(value.trim())?, + state: PhantomData, + }) + } + + pub(crate) fn complete_path( + mut self, + target: &[&str], + ) -> Result, ApiUrlError> { + let existing: Vec = self + .url + .path_segments() + .ok_or(ApiUrlError::CannotBeBase)? + .filter(|segment| !segment.is_empty()) + .map(str::to_string) + .collect(); + let overlap = (0..=existing.len().min(target.len())) + .rev() + .find(|&length| { + existing[existing.len() - length..] + .iter() + .map(String::as_str) + .eq(target[..length].iter().copied()) + }) + .unwrap_or(0); + self.url + .path_segments_mut() + .map_err(|()| ApiUrlError::CannotBeBase)? + .pop_if_empty() + .extend(target[overlap..].iter().copied()); + Ok(ApiUrl { + url: self.url, + state: PhantomData, + }) + } +} + +impl ApiUrl { + pub(crate) fn into_string(self) -> String { + self.url.into() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn completion_appends_only_the_missing_path_suffix() { + for (base, expected) in [ + ("https://example.test", "https://example.test/v1/ocr"), + ("https://example.test/v1", "https://example.test/v1/ocr"), + ("https://example.test/v1/ocr", "https://example.test/v1/ocr"), + ] { + let actual = ApiUrl::parse(base) + .and_then(|url| url.complete_path(&["v1", "ocr"])) + .map(|url| url.into_string()) + .expect("url builds"); + assert_eq!(actual, expected); + } + } + + #[test] + fn completion_places_paths_before_queries() { + let actual = ApiUrl::parse("https://example.test/v1?tenant=a") + .and_then(|url| url.complete_path(&["v1", "ocr"])) + .map(|url| url.into_string()) + .expect("url builds"); + assert_eq!(actual, "https://example.test/v1/ocr?tenant=a"); + } +} diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs new file mode 100644 index 00000000000..d1828dfb816 --- /dev/null +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -0,0 +1,227 @@ +use std::sync::{Arc, Mutex}; + +use serde_json::{Value, json}; + +use super::OcrClient; +use super::hooks::{OcrHookFuture, OcrHooks, OcrLogFuture, OcrPreCallRequest}; +use super::test_support::{MockResponse, mock_server, perform_ocr, wire_request}; +use super::wire::{OcrWireRequest, decode_request}; +use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleTiming}; + +#[test] +fn request_boundary_selects_mistral_and_rejects_unknown_providers() { + let request = OcrWireRequest { + model: "mistral/model".into(), + document: json!({"type":"document_url","document_url":"https://example.com/doc.pdf"}), + api_key: Some("key".into()), + api_base: None, + custom_llm_provider: None, + extra_headers: None, + optional_params: json!({"extract_header":true,"unknown":42}) + .as_object() + .unwrap() + .clone(), + timeout_seconds: None, + }; + assert!(decode_request(request).is_ok()); + assert!( + decode_request(OcrWireRequest { + model: "model".into(), + document: json!({"type":"document_url","document_url":"https://example.com/doc.pdf"}), + api_key: Some("key".into()), + api_base: None, + custom_llm_provider: Some("unknown".into()), + extra_headers: None, + optional_params: serde_json::Map::new(), + timeout_seconds: None, + }) + .is_err() + ); +} + +#[tokio::test] +async fn facade_executes_direct_mistral_once() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ + "pages":[{"index":0,"markdown":"hello","custom":"preserved"}], + "usage_info":{"pages_processed":1} + }))]) + .await; + let result = perform_ocr(wire_request( + "mistral/model", + &base, + json!({"extract_header":true,"unknown":"ignored"}), + )) + .await + .unwrap(); + server.await.unwrap(); + assert_eq!(result.pages[0]["markdown"], "hello"); + assert_eq!(result.pages[0]["custom"], "preserved"); + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with("POST /v1/ocr ")); + assert!( + requests[0] + .to_ascii_lowercase() + .contains("authorization: bearer test-key\r\n") + ); + let body: Value = serde_json::from_str(requests[0].split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!( + body, + json!({ + "model":"model", + "document":{"type":"document_url","document_url":"data:application/pdf;base64,YWJj"}, + "extract_header":true + }) + ); +} + +#[tokio::test] +async fn facade_retains_native_response_when_requested() { + let provider_response = json!({ + "pages":[{"index":0,"markdown":"hello"}], + "usage_info":{"pages_processed":1}, + "provider_only":"preserved" + }); + let (base, _, server) = mock_server(vec![MockResponse::json(provider_response.clone())]).await; + let response = perform_ocr(wire_request( + "mistral/model", + &base, + json!({"req_format":"native"}), + )) + .await + .unwrap(); + + server.await.unwrap(); + assert_eq!(response.provider_native_response, Some(provider_response)); +} + +#[tokio::test] +async fn facade_uses_the_injected_http_client() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; + let mut default_headers = reqwest::header::HeaderMap::new(); + default_headers.insert( + "x-transport-owner", + reqwest::header::HeaderValue::from_static("host"), + ); + let provider_http = reqwest::Client::builder() + .default_headers(default_headers) + .build() + .unwrap(); + OcrClient::new(provider_http) + .unwrap() + .perform(wire_request("mistral/model", &base, json!({}))) + .await + .unwrap(); + server.await.unwrap(); + assert!(seen.lock().unwrap()[0].contains("x-transport-owner: host")); +} + +struct RecordingHooks { + events: Arc>>, + block: bool, +} + +impl OcrHooks for RecordingHooks { + fn has_guardrails(&self) -> bool { + true + } + + fn pre_call(&self, request: OcrPreCallRequest) -> OcrHookFuture<'_, OcrPreCallRequest> { + Box::pin(async move { + self.events.lock().unwrap().push("pre"); + if self.block { + return Err(crate::Error::InvalidRequest("blocked".into())); + } + Ok(request) + }) + } + + fn during_call( + &self, + request: super::hooks::OcrDuringCallRequest, + ) -> OcrHookFuture<'_, super::hooks::OcrDuringCallRequest> { + Box::pin(async move { + self.events.lock().unwrap().push("during"); + Ok(request) + }) + } + + fn success<'a>( + &'a self, + _context: &'a CallLifecycleContext, + _response: &'a super::LiteLLMOcrResponse, + _timing: &'a CallLifecycleTiming, + ) -> OcrLogFuture<'a> { + Box::pin(async move { + self.events.lock().unwrap().push("success"); + }) + } + + fn failure<'a>( + &'a self, + _context: &'a CallLifecycleContext, + _error: &'a crate::Error, + _timing: &'a CallLifecycleTiming, + ) -> OcrLogFuture<'a> { + Box::pin(async move { + self.events.lock().unwrap().push("failure"); + }) + } +} + +#[tokio::test] +async fn lifecycle_orders_hooks_and_emits_one_success() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; + let events = Arc::new(Mutex::new(Vec::new())); + let request = wire_request("mistral/model", &base, json!({})); + let request = super::LiteLLMOcrRequest { + hooks: Arc::new(RecordingHooks { + events: events.clone(), + block: false, + }), + ..request + }; + perform_ocr(request).await.unwrap(); + server.await.unwrap(); + assert_eq!(*events.lock().unwrap(), ["pre", "during", "success"]); + assert_eq!(seen.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn lifecycle_blocking_prevents_execution_and_emits_one_failure() { + let events = Arc::new(Mutex::new(Vec::new())); + let request = wire_request("mistral/model", "http://127.0.0.1:1", json!({})); + let request = super::LiteLLMOcrRequest { + hooks: Arc::new(RecordingHooks { + events: events.clone(), + block: true, + }), + ..request + }; + let error = perform_ocr(request).await.unwrap_err(); + assert!(matches!(error, crate::Error::InvalidRequest(_))); + assert_eq!(*events.lock().unwrap(), ["pre", "failure"]); +} + +#[tokio::test] +async fn upstream_failure_emits_one_terminal_failure() { + let (base, seen, server) = mock_server(vec![MockResponse { + status: 500, + headers: vec![], + body: json!({"error":"failed"}), + }]) + .await; + let events = Arc::new(Mutex::new(Vec::new())); + let request = wire_request("mistral/model", &base, json!({})); + let request = super::LiteLLMOcrRequest { + hooks: Arc::new(RecordingHooks { + events: events.clone(), + block: false, + }), + ..request + }; + assert!(perform_ocr(request).await.is_err()); + server.await.unwrap(); + assert_eq!(*events.lock().unwrap(), ["pre", "during", "failure"]); + assert_eq!(seen.lock().unwrap().len(), 1); +} diff --git a/litellm-rust/crates/core/tests/ocr/support.rs b/litellm-rust/crates/core/tests/ocr/support.rs new file mode 100644 index 00000000000..d9720019419 --- /dev/null +++ b/litellm-rust/crates/core/tests/ocr/support.rs @@ -0,0 +1,106 @@ +use std::sync::{Arc, Mutex}; + +use serde_json::{Value, json}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; + +use crate::ocr::wire::{OcrWireRequest, decode_request}; +use crate::ocr::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrClient}; + +pub(crate) fn ocr_client() -> OcrClient { + OcrClient::for_test(reqwest::Client::new()) +} + +pub(crate) async fn perform_ocr( + request: LiteLLMOcrRequest, +) -> Result { + ocr_client().perform(request).await +} + +pub(crate) fn wire_request(model: &str, base: &str, options: Value) -> LiteLLMOcrRequest { + decode_request(OcrWireRequest { + model: model.into(), + document: json!({"type":"document_url","document_url":"data:application/pdf;base64,YWJj"}), + api_key: Some("test-key".into()), + api_base: Some(base.into()), + custom_llm_provider: None, + extra_headers: None, + optional_params: options.as_object().unwrap().clone(), + timeout_seconds: Some(2.0), + }) + .unwrap() +} + +pub(crate) struct MockResponse { + pub status: u16, + pub headers: Vec<(&'static str, String)>, + pub body: Value, +} + +impl MockResponse { + pub fn json(body: Value) -> Self { + Self { + status: 200, + headers: vec![], + body, + } + } +} + +pub(crate) async fn mock_server( + responses: Vec, +) -> (String, Arc>>, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let requests = Arc::new(Mutex::new(Vec::new())); + let seen = requests.clone(); + let server_base = base.clone(); + let task = tokio::spawn(async move { + for response in responses { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut bytes = Vec::new(); + let mut buffer = [0u8; 4096]; + let header_end = loop { + let n = socket.read(&mut buffer).await.unwrap(); + assert!(n > 0); + bytes.extend_from_slice(&buffer[..n]); + if let Some(index) = bytes.windows(4).position(|s| s == b"\r\n\r\n") { + break index + 4; + } + }; + let length = String::from_utf8_lossy(&bytes[..header_end]) + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().unwrap()) + }) + .unwrap_or(0); + while bytes.len() < header_end + length { + let n = socket.read(&mut buffer).await.unwrap(); + assert!(n > 0); + bytes.extend_from_slice(&buffer[..n]); + } + seen.lock() + .unwrap() + .push(String::from_utf8_lossy(&bytes).into_owned()); + let body = serde_json::to_vec(&response.body).unwrap(); + let headers = response + .headers + .into_iter() + .map(|(name, value)| { + format!("{name}: {}\r\n", value.replace("{base}", &server_base)) + }) + .collect::(); + let head = format!( + "HTTP/1.1 {} OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n{}\r\n", + response.status, + body.len(), + headers + ); + socket.write_all(head.as_bytes()).await.unwrap(); + socket.write_all(&body).await.unwrap(); + } + }); + (base, requests, task) +} diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index 76c298abf89..407475dedce 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -41,6 +41,7 @@ pub(crate) fn chat_completions_error_to_pyerr(err: Error) -> PyErr { | Error::InvalidRequest(_) | Error::InvalidType { .. } | Error::MissingField(_) + | Error::MissingApiKey { .. } | Error::Routing(_) // Nothing reached the provider, so serving it on Python cannot double // bill and is the only way the caller gets an answer at all. From 93bd3d61b56a167b794c6b8cc2dcc6dcefaf4a47 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:20:49 -0700 Subject: [PATCH 059/126] fix(mcp): preserve verified identity when warming OAuth cache --- .../mcp_server/discoverable_endpoints.py | 1 + .../mcp_server/test_discoverable_endpoints.py | 43 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index c8e02be308b..920fe17e43e 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -634,6 +634,7 @@ async def _store_per_user_token_server_side( server_id=server.server_id, access_token=access_token, ttl=ttl, + identity_binding_proof=identity_binding_proof, ) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 42187cb1771..ff649075468 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -11103,3 +11103,46 @@ async def test_identity_bound_authorization_carries_nonce_and_caller_through_cal assert sealed.upstream_code == "upstream-code" assert sealed.oauth_nonce == query["nonce"][0] assert returned["state"] == ["client-state"] + + +@pytest.mark.asyncio +async def test_enforced_login_warms_verified_token_readable_without_database_lookup(monkeypatch): + from types import SimpleNamespace + from litellm.caching.caching import DualCache + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import db, mcp_server_manager + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import _store_per_user_token_server_side + from litellm.proxy._experimental.mcp_server.oauth2_token_cache import mcp_per_user_token_cache + from litellm.proxy._experimental.mcp_server.oauth_identity_binding import current_binding_proof + from litellm.proxy._experimental.mcp_server.outbound_credentials.dual_cache_token_backend import DualCacheTokenCacheBackend + from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import CachedOAuthTokenStore + from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import LazyPerUserOAuthTokenStore + from litellm.proxy._experimental.mcp_server.outbound_credentials.v2_token_store import V2PerUserTokenStore + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="srv", name="srv", transport="http", auth_type="oauth2", + oauth_identity_binding={"mode": "enforce", "issuer": "https://idp.example", "audiences": ["client"], + "caller_field": "user_id", "principal_claim": "sub"}, + ) + proof = await current_binding_proof(server.oauth_identity_binding, "alice", "srv") + cache = DualCache() + monkeypatch.setenv("LITELLM_SALT_KEY", "test-cache-warm-encryption-salt") + monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", SimpleNamespace(invalidate_user_oauth_token_cache=AsyncMock())) + monkeypatch.setattr(db, "store_user_oauth_credential", AsyncMock()) + await _store_per_user_token_server_side( + server=server, user_id="alice", token_response={"access_token": "alice-token", "refresh_token": "private", "expires_in": 3600}, + identity_binding_proof=proof, + ) + read = AsyncMock(return_value=None) + cached = CachedOAuthTokenStore( + V2PerUserTokenStore(read), default_ttl_seconds=300, backend=DualCacheTokenCacheBackend(cache, mcp_per_user_token_cache._codec()), + ) + store = LazyPerUserOAuthTokenStore(lambda _: server, store_builder=lambda _: (cached, True), redis_available=lambda: True) + token = await store.fetch("alice", "srv") + assert token is not None and token.access_token == "alice-token" + assert token.identity_binding_proof == proof + assert token.refresh_token is None + read.assert_not_awaited() From 126dc37de412df372604814b6674fab45635760d Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 13:33:36 -0700 Subject: [PATCH 060/126] fix(logging): reconstruct classifier audit redaction payloads --- litellm/integrations/custom_logger.py | 46 ++++------ .../litellm_core_utils/classifier_logging.py | 2 +- litellm/litellm_core_utils/redact_messages.py | 24 ++--- .../spend_tracking/spend_tracking_utils.py | 4 +- .../test_redact_messages.py | 90 ++++++++++++++----- 5 files changed, 96 insertions(+), 70 deletions(-) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 473bf223105..df3237bff07 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -897,10 +897,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac This is useful for logging payloads that contain sensitive information. """ - from copy import copy - import litellm from litellm import Choices, Message, ModelResponse + from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit turn_off_message_logging: Final[bool] = getattr(self, "turn_off_message_logging", False) excluded_fields: Final[list[str] | None] = getattr(litellm, "standard_logging_payload_excluded_fields", None) @@ -909,41 +908,25 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac if turn_off_message_logging is False and not excluded_fields: return model_call_details - # Only make a shallow copy of the top-level dict to avoid deepcopy issues - # with complex objects like AuthenticationError that may be present - model_call_details_copy: Final = copy(model_call_details) standard_logging_object: Final = model_call_details.get("standard_logging_object") if standard_logging_object is None: - return model_call_details_copy + return model_call_details.copy() # Make a copy of just the standard_logging_object to avoid modifying the original - standard_logging_object_copy: Final = copy(standard_logging_object) - - # Handle excluded fields - remove them entirely from the payload - if excluded_fields: - for field in excluded_fields: - if field in standard_logging_object_copy: - del standard_logging_object_copy[field] + standard_logging_object_copy: Final = { + key: value + for key, value in standard_logging_object.items() + if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS) + } # Handle turn_off_message_logging - redact messages and responses (if not already excluded) if turn_off_message_logging: - from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit - - for field in CLASSIFIER_AUDIT_FIELDS: - standard_logging_object_copy.pop(field, None) - params: Final = model_call_details_copy.get("litellm_params") - request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None - if isinstance(params, dict) and isinstance(request, dict): - model_call_details_copy["litellm_params"] = { - **params, - "proxy_server_request": without_classifier_audit(request), - } redacted_str: Final = "redacted-by-litellm" - if "messages" not in (excluded_fields or []) and standard_logging_object_copy.get("messages") is not None: + if "messages" not in (excluded_fields or ()) and standard_logging_object_copy.get("messages") is not None: standard_logging_object_copy["messages"] = [Message(content=redacted_str).model_dump()] - if "response" not in (excluded_fields or []) and standard_logging_object_copy.get("response") is not None: + if "response" not in (excluded_fields or ()) and standard_logging_object_copy.get("response") is not None: response: Final = standard_logging_object_copy["response"] # Check if this is a ResponsesAPIResponse (has "output" field) if isinstance(response, dict) and "output" in response: @@ -967,8 +950,15 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac model_response_dict: Final = model_response.model_dump() standard_logging_object_copy["response"] = model_response_dict - model_call_details_copy["standard_logging_object"] = standard_logging_object_copy - return model_call_details_copy + redacted_details: Final = {**model_call_details, "standard_logging_object": standard_logging_object_copy} + params: Final = model_call_details.get("litellm_params") + request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None + if turn_off_message_logging and isinstance(params, dict) and isinstance(request, dict): + return { + **redacted_details, + "litellm_params": {**params, "proxy_server_request": without_classifier_audit(request)}, + } + return redacted_details async def get_proxy_server_request_from_cold_storage_with_object_key( self, diff --git a/litellm/litellm_core_utils/classifier_logging.py b/litellm/litellm_core_utils/classifier_logging.py index c0c93161099..4cbfdb8cd49 100644 --- a/litellm/litellm_core_utils/classifier_logging.py +++ b/litellm/litellm_core_utils/classifier_logging.py @@ -38,7 +38,7 @@ def is_classifier_call(call_type: str, params: Mapping[str, object]) -> bool: def masked_originating_request(request_kwargs: Mapping[str, object] | None) -> Mapping[str, JsonValue] | None: - request: Final = (request_kwargs or {}).get("proxy_server_request") + request: Final = request_kwargs.get("proxy_server_request") if request_kwargs is not None else None body: Final = request.get("body") if isinstance(request, Mapping) else None if not isinstance(body, Mapping): return None diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index c97566c21a8..9d22a5ddef5 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -16,7 +16,7 @@ from typing import TYPE_CHECKING, Any, Final import litellm from litellm.constants import REDACTED_BY_LITELLM from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit +from litellm.litellm_core_utils.classifier_logging import without_classifier_audit from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, ) @@ -171,22 +171,13 @@ def redacted_standard_logging_payload(payload: Mapping[str, object]) -> Mapping[ payload, but the failure path does not, so a callback that batches both has to redact the ones it is handed. """ - redacted: Final = copy.deepcopy(dict(payload)) # mutable-ok: redacted in place below - _redact_standard_logging_object({"standard_logging_object": redacted}) # mutable-ok: the callee's shape - return redacted + return _redact_standard_logging_object(payload) -def _redact_standard_logging_object(model_call_details: dict): - """Redact messages and response inside standard_logging_object if present.""" - standard_logging_object: Final = model_call_details.get("standard_logging_object") - if standard_logging_object is None: - return - +def _redact_standard_logging_object(payload: Mapping[str, object]) -> dict[str, object]: + standard_logging_object: Final = copy.deepcopy(without_classifier_audit(payload)) redacted_str: Final = REDACTED_BY_LITELLM - for field in CLASSIFIER_AUDIT_FIELDS: - standard_logging_object.pop(field, None) - if standard_logging_object.get("messages") is not None: standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}] @@ -207,6 +198,7 @@ def _redact_standard_logging_object(model_call_details: dict): else: # For other formats (empty dict, None, etc.), use simple text format standard_logging_object["response"] = {"text": redacted_str} + return standard_logging_object def _redact_tool_calls_dict(message: Mapping[str, object]) -> None: @@ -258,8 +250,6 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons copy via redact_streaming_responses_for_custom_logger instead. """ # Redact model_call_details - for field in CLASSIFIER_AUDIT_FIELDS: - model_call_details.pop(field, None) params: Final = model_call_details.get("litellm_params") request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None if isinstance(params, dict) and isinstance(request, Mapping): @@ -267,7 +257,9 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}] model_call_details["prompt"] = "" model_call_details["input"] = "" - _redact_standard_logging_object(model_call_details) + standard_logging_object: Final = model_call_details.get("standard_logging_object") + if isinstance(standard_logging_object, Mapping): + model_call_details["standard_logging_object"] = _redact_standard_logging_object(standard_logging_object) redact_vertex_ai_metadata_from_litellm_params(model_call_details) # Redact streaming response diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index da55d1cb153..360b24575bf 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -22,7 +22,7 @@ from litellm.constants import ( from litellm.constants import ( MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB, ) -from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields +from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields, without_classifier_audit from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, reconstruct_model_name, @@ -1277,7 +1277,7 @@ def _get_proxy_server_request_for_spend_logs_payload( # If redaction is enabled, convert to serializable dict before redacting if should_redact_message_logging(model_call_details=model_call_details): - _request_body = _convert_mapping_to_json_serializable(_request_body) + _request_body = _convert_mapping_to_json_serializable(without_classifier_audit(_request_body)) perform_redaction(model_call_details=_request_body, result=None) _request_body = _sanitize_request_body_for_spend_logs_payload(_request_body) diff --git a/tests/test_litellm/litellm_core_utils/test_redact_messages.py b/tests/test_litellm/litellm_core_utils/test_redact_messages.py index 1ba9901aab0..76092e96307 100644 --- a/tests/test_litellm/litellm_core_utils/test_redact_messages.py +++ b/tests/test_litellm/litellm_core_utils/test_redact_messages.py @@ -17,6 +17,7 @@ from litellm.litellm_core_utils.redact_messages import ( _redact_responses_api_output, perform_redaction, redact_streaming_responses_for_custom_logger, + redacted_standard_logging_payload, should_redact_message_logging, ) from litellm.responses.main import mock_responses_api_response @@ -777,29 +778,6 @@ class TestPerformRedaction: assert response_obj.choices[0].message.content == "secret content" - -@pytest.mark.parametrize("callback_only", [False, True]) -def test_classifier_audit_redaction_removes_both_fields_and_source_carrier(callback_only: bool) -> None: - audit: Final = {"classifier_input": {"system": "private rubric"}, "originating_request_masked": {"input": "private source"}} - details: Final = { - "standard_logging_object": {**audit, "messages": [], "response": {}}, - "litellm_params": {"proxy_server_request": {"body": {}, "originating_request_masked": audit["originating_request_masked"]}}, - } - logger: Final = CustomLogger() - logger.turn_off_message_logging = True - if callback_only: - redacted: Final = logger.redact_standard_logging_payload_from_model_call_details(details) - assert "classifier_input" not in redacted["standard_logging_object"] - assert "originating_request_masked" not in redacted["standard_logging_object"] - assert "originating_request_masked" not in redacted["litellm_params"]["proxy_server_request"] - assert details["standard_logging_object"]["classifier_input"] == audit["classifier_input"] - assert details["litellm_params"]["proxy_server_request"]["originating_request_masked"] == audit["originating_request_masked"] - else: - perform_redaction(details, result=None) - assert "classifier_input" not in details["standard_logging_object"] - assert "originating_request_masked" not in details["standard_logging_object"] - assert "originating_request_masked" not in details["litellm_params"]["proxy_server_request"] - def test_unredactable_result_is_not_deepcopied(self): """A result shape no branch can redact must not be deepcopied. @@ -882,3 +860,69 @@ class TestRedactStreamingResponsesForCustomLogger: assert result_details is model_call_details assert response_obj.choices[0].message.content == "secret content" + + +@pytest.mark.parametrize("callback_only", [False, True]) +def test_classifier_audit_redaction_removes_both_fields_and_source_carrier(callback_only: bool) -> None: + audit: Final = {"classifier_input": {"system": "private rubric"}, "originating_request_masked": {"input": "private source"}} + standard_payload: Final = { + **audit, + "messages": [{"role": "user", "content": "private prompt"}], + "response": {"choices": [{"message": {"content": "private answer"}}]}, + "model": "classifier", + } + details: Final = { + "standard_logging_object": standard_payload, + "litellm_params": {"proxy_server_request": {"body": {}, "originating_request_masked": audit["originating_request_masked"]}}, + } + logger: Final = CustomLogger() + logger.turn_off_message_logging = True + if callback_only: + redacted: Final = logger.redact_standard_logging_payload_from_model_call_details(details) + assert "classifier_input" not in redacted["standard_logging_object"] + assert "originating_request_masked" not in redacted["standard_logging_object"] + assert "originating_request_masked" not in redacted["litellm_params"]["proxy_server_request"] + assert details["standard_logging_object"]["classifier_input"] == audit["classifier_input"] + assert details["litellm_params"]["proxy_server_request"]["originating_request_masked"] == audit["originating_request_masked"] + else: + perform_redaction(details, result=None) + assert "classifier_input" not in details["standard_logging_object"] + assert "originating_request_masked" not in details["standard_logging_object"] + assert "originating_request_masked" not in details["litellm_params"]["proxy_server_request"] + + assert standard_payload["classifier_input"] == audit["classifier_input"] + assert standard_payload["originating_request_masked"] == audit["originating_request_masked"] + assert standard_payload["messages"][0]["content"] == "private prompt" + assert standard_payload["response"]["choices"][0]["message"]["content"] == "private answer" + + +@pytest.mark.parametrize("excluded", [False, True]) +def test_classifier_callback_redaction_preserves_exclusions(monkeypatch: pytest.MonkeyPatch, excluded: bool) -> None: + monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["messages", "response"] if excluded else []) + payload: Final = { + "classifier_input": {"system": "private rubric"}, + "originating_request_masked": {"input": "private source"}, + "messages": [{"role": "user", "content": "private prompt"}], + "response": {"choices": [{"message": {"content": "private answer"}}]}, + "model": "classifier", + } + logger: Final = CustomLogger() + logger.turn_off_message_logging = True + redacted: Final = logger.redact_standard_logging_payload_from_model_call_details({"standard_logging_object": payload}) + stored: Final = redacted["standard_logging_object"] + assert "classifier_input" not in stored + assert "originating_request_masked" not in stored + assert stored["model"] == "classifier" + assert ("messages" not in stored) is excluded + assert ("response" not in stored) is excluded + if not excluded: + assert stored["messages"][0]["content"] == "redacted-by-litellm" + assert stored["response"]["choices"][0]["message"]["content"] == "redacted-by-litellm" + + failure_payload: Final = redacted_standard_logging_payload(payload) + assert "classifier_input" not in failure_payload + assert "originating_request_masked" not in failure_payload + assert failure_payload["messages"][0]["content"] == "redacted-by-litellm" + assert failure_payload["response"]["choices"][0]["message"]["content"] == "redacted-by-litellm" + assert payload["classifier_input"] == {"system": "private rubric"} + assert payload["response"]["choices"][0]["message"]["content"] == "private answer" From a9cec50960e76f1e311bbbb679d6b3dd3394c93e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:36:23 -0700 Subject: [PATCH 061/126] feat(infra): scale gateway on per-pod RPS and TPS in Helm and Terraform (#40479) * feat(infra): scale gateway on per-pod RPM and TPM in Helm and Terraform Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(helm): require the metrics server before rendering the gateway ServiceMonitor The http port serves /metrics/ behind virtual-key auth, so a ServiceMonitor pointed at it only collects 401s and the RPM/TPM HPA metrics never appear Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(infra): express gateway HPA, KEDA and ECS workload targets per second Rename the per-pod request and token targets in both Helm charts and the AWS module from per minute to per second, and shorten the recommended Prometheus rate window to [1m] with no * 60 so the adapter and KEDA signals are what the HPA compares against. ECS keeps CloudWatch's 60-second aggregation: the ALB target is 60x the per-second variable and the token metric math divides the period Sum by 60 before dividing by the running task count. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- helm/litellm-helm/templates/hpa.yaml | 18 ++ helm/litellm-helm/templates/keda.yaml | 21 ++ helm/litellm-helm/tests/hpa_tests.yaml | 78 +++++++ helm/litellm-helm/tests/keda_tests.yaml | 106 +++++++++ helm/litellm-helm/values.yaml | 36 +++ helm/litellm/templates/gateway/hpa.yaml | 18 ++ .../templates/gateway/servicemonitor.yaml | 28 +++ .../tests/hpa_workload_metrics_tests.yaml | 213 ++++++++++++++++++ helm/litellm/values.yaml | 29 +++ terraform/litellm/aws/README.md | 55 +++++ terraform/litellm/aws/autoscaling.tf | 99 ++++++++ .../aws/tests/workload_autoscaling.tftest.hcl | 194 ++++++++++++++++ terraform/litellm/aws/variables.tf | 41 ++++ terraform/litellm/gcp/README.md | 18 ++ 14 files changed, 954 insertions(+) create mode 100644 helm/litellm-helm/tests/keda_tests.yaml create mode 100644 helm/litellm/templates/gateway/servicemonitor.yaml create mode 100644 helm/litellm/tests/hpa_workload_metrics_tests.yaml create mode 100644 terraform/litellm/aws/tests/workload_autoscaling.tftest.hcl diff --git a/helm/litellm-helm/templates/hpa.yaml b/helm/litellm-helm/templates/hpa.yaml index fec4d1f5c5e..010c4095ab3 100644 --- a/helm/litellm-helm/templates/hpa.yaml +++ b/helm/litellm-helm/templates/hpa.yaml @@ -33,4 +33,22 @@ spec: type: Utilization averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} + {{- with .Values.autoscaling.targetRequestsPerSecond }} + - type: Pods + pods: + metric: + name: litellm_requests_per_second + target: + type: AverageValue + averageValue: {{ toJson . | trimAll "\"" | quote }} + {{- end }} + {{- with .Values.autoscaling.targetTokensPerSecond }} + - type: Pods + pods: + metric: + name: litellm_tokens_per_second + target: + type: AverageValue + averageValue: {{ toJson . | trimAll "\"" | quote }} + {{- end }} {{- end }} diff --git a/helm/litellm-helm/templates/keda.yaml b/helm/litellm-helm/templates/keda.yaml index fe5190fffc6..bf585d0d4be 100644 --- a/helm/litellm-helm/templates/keda.yaml +++ b/helm/litellm-helm/templates/keda.yaml @@ -23,6 +23,27 @@ spec: triggers: {{- with .Values.keda.triggers }} {{- toYaml . | nindent 2 }} +{{- end }} +{{- $prom := .Values.keda.prometheus }} +{{- if or $prom.requestsPerSecond $prom.tokensPerSecond }} +{{- if not $prom.serverAddress }} +{{- fail "keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set" }} +{{- end }} +{{- $selector := printf "namespace=%q,job=%q" .Release.Namespace (printf "%s%s" (include "litellm.fullname" .) (ternary "-metrics" "" .Values.metricsServer.enabled)) }} +{{- with $prom.requestsPerSecond }} + - type: prometheus + metadata: + serverAddress: {{ $prom.serverAddress | quote }} + threshold: {{ toJson . | trimAll "\"" | quote }} + query: {{ printf "sum(rate(litellm_proxy_total_requests_metric_total{%s}[1m]))" $selector | quote }} +{{- end }} +{{- with $prom.tokensPerSecond }} + - type: prometheus + metadata: + serverAddress: {{ $prom.serverAddress | quote }} + threshold: {{ toJson . | trimAll "\"" | quote }} + query: {{ printf "sum(rate(litellm_total_tokens_metric_total{%s}[1m]))" $selector | quote }} +{{- end }} {{- end }} advanced: restoreToOriginalReplicaCount: {{ .Values.keda.restoreToOriginalReplicaCount }} diff --git a/helm/litellm-helm/tests/hpa_tests.yaml b/helm/litellm-helm/tests/hpa_tests.yaml index cd062dd5971..e446f58c8fe 100644 --- a/helm/litellm-helm/tests/hpa_tests.yaml +++ b/helm/litellm-helm/tests/hpa_tests.yaml @@ -61,6 +61,84 @@ tests: - equal: { path: "spec.metrics[1].resource.name", value: memory } - equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 } + - it: "renders no workload metrics by default" + set: + autoscaling.enabled: true + autoscaling.targetMemoryUtilizationPercentage: 80 + asserts: + - lengthEqual: { path: spec.metrics, count: 2 } + - notContains: { path: spec.metrics, content: { type: Pods }, any: true } + + - it: "adds a requests-per-second Pods metric after the cpu metric" + set: + autoscaling.enabled: true + autoscaling.targetRequestsPerSecond: 90 + asserts: + - lengthEqual: { path: spec.metrics, count: 2 } + - equal: { path: "spec.metrics[0].resource.name", value: cpu } + - equal: + path: "spec.metrics[1]" + value: + type: Pods + pods: + metric: { name: litellm_requests_per_second } + target: { type: AverageValue, averageValue: "90" } + + - it: "adds a tokens-per-second Pods metric on its own" + set: + autoscaling.enabled: true + autoscaling.targetTokensPerSecond: 6M + asserts: + - lengthEqual: { path: spec.metrics, count: 2 } + - equal: + path: "spec.metrics[1]" + value: + type: Pods + pods: + metric: { name: litellm_tokens_per_second } + target: { type: AverageValue, averageValue: "6M" } + - notContains: + path: spec.metrics + content: { type: Pods, pods: { metric: { name: litellm_requests_per_second } } } + any: true + + - it: "renders requests, tokens, cpu and memory metrics together" + set: + autoscaling.enabled: true + autoscaling.targetMemoryUtilizationPercentage: 80 + autoscaling.targetRequestsPerSecond: 90 + autoscaling.targetTokensPerSecond: 6000000 + asserts: + - lengthEqual: { path: spec.metrics, count: 4 } + - equal: { path: "spec.metrics[0].resource.name", value: cpu } + - equal: { path: "spec.metrics[1].resource.name", value: memory } + - equal: { path: "spec.metrics[2].pods.metric.name", value: litellm_requests_per_second } + - equal: { path: "spec.metrics[2].pods.target.averageValue", value: "90" } + - equal: { path: "spec.metrics[3].pods.metric.name", value: litellm_tokens_per_second } + - equal: { path: "spec.metrics[3].pods.target.averageValue", value: "6000000" } + + - it: "scales on workload metrics alone when the cpu target is cleared" + set: + autoscaling.enabled: true + autoscaling.targetCPUUtilizationPercentage: null + autoscaling.targetRequestsPerSecond: 90 + autoscaling.targetTokensPerSecond: 6000000 + asserts: + - lengthEqual: { path: spec.metrics, count: 2 } + - notContains: { path: spec.metrics, content: { type: Resource }, any: true } + - equal: { path: "spec.metrics[0].pods.metric.name", value: litellm_requests_per_second } + - equal: { path: "spec.metrics[1].pods.metric.name", value: litellm_tokens_per_second } + - notMatchRegexRaw: { pattern: per_minute } + + - it: "ignores the per-minute keys, which the chart never shipped" + set: + autoscaling.enabled: true + autoscaling.targetRequestsPerMinute: 5400 + autoscaling.targetTokensPerMinute: 360000000 + asserts: + - lengthEqual: { path: spec.metrics, count: 1 } + - notContains: { path: spec.metrics, content: { type: Pods }, any: true } + - it: "renders no hpa when autoscaling is disabled" asserts: - hasDocuments: { count: 0 } diff --git a/helm/litellm-helm/tests/keda_tests.yaml b/helm/litellm-helm/tests/keda_tests.yaml new file mode 100644 index 00000000000..c9598646223 --- /dev/null +++ b/helm/litellm-helm/tests/keda_tests.yaml @@ -0,0 +1,106 @@ +suite: "keda" +templates: + - keda.yaml +release: + name: rel + namespace: llm +tests: + - it: "renders no scaled object by default" + asserts: + - hasDocuments: { count: 0 } + + - it: "passes user triggers through and adds no prometheus triggers by default" + set: + keda.enabled: true + keda.triggers: + - type: cpu + metricType: Utilization + metadata: { value: "60" } + asserts: + - isKind: { of: ScaledObject } + - equal: + path: spec.triggers + value: + - type: cpu + metricType: Utilization + metadata: { value: "60" } + + - it: "scales on release-wide requests per second divided by the per-replica target" + set: + keda.enabled: true + keda.prometheus.serverAddress: http://prometheus-operated.monitoring.svc:9090 + keda.prometheus.requestsPerSecond: 90 + asserts: + - lengthEqual: { path: spec.triggers, count: 1 } + - equal: + path: "spec.triggers[0]" + value: + type: prometheus + metadata: + serverAddress: http://prometheus-operated.monitoring.svc:9090 + threshold: "90" + query: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm"}[1m])) + + - it: "scales on tokens per second on its own" + set: + keda.enabled: true + keda.prometheus.serverAddress: http://prom:9090 + keda.prometheus.tokensPerSecond: 6000000 + asserts: + - lengthEqual: { path: spec.triggers, count: 1 } + - equal: { path: "spec.triggers[0].type", value: prometheus } + - equal: { path: "spec.triggers[0].metadata.threshold", value: "6000000" } + - equal: + path: "spec.triggers[0].metadata.query" + value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm"}[1m])) + + - it: "appends requests and tokens triggers after user triggers and selects the metrics service job" + set: + keda.enabled: true + metricsServer.enabled: true + keda.triggers: + - type: cpu + metricType: Utilization + metadata: { value: "60" } + keda.prometheus.serverAddress: http://prom:9090 + keda.prometheus.requestsPerSecond: 90 + keda.prometheus.tokensPerSecond: 6000000 + asserts: + - lengthEqual: { path: spec.triggers, count: 3 } + - equal: { path: "spec.triggers[0].type", value: cpu } + - equal: { path: "spec.triggers[1].metadata.threshold", value: "90" } + - equal: + path: "spec.triggers[1].metadata.query" + value: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m])) + - equal: { path: "spec.triggers[2].metadata.threshold", value: "6000000" } + - equal: + path: "spec.triggers[2].metadata.query" + value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m])) + - notMatchRegexRaw: { pattern: "\\* *60|per_minute|PerMinute" } + + - it: "ignores the per-minute keys, which the chart never shipped" + set: + keda.enabled: true + keda.prometheus.serverAddress: http://prom:9090 + keda.prometheus.requestsPerMinute: 5400 + keda.prometheus.tokensPerMinute: 360000000 + asserts: + - isKind: { of: ScaledObject } + - isNullOrEmpty: { path: spec.triggers } + + - it: "refuses a workload target without a prometheus server address" + set: + keda.enabled: true + keda.prometheus.requestsPerSecond: 90 + asserts: + - failedTemplate: + errorMessage: keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set + + - it: "yields to the hpa when both autoscalers are enabled" + set: + autoscaling.enabled: true + keda.enabled: true + keda.prometheus.serverAddress: http://prom:9090 + keda.prometheus.requestsPerSecond: 90 + asserts: + - hasDocuments: { count: 0 } diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 8dc7b967e11..907f85de8fc 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -222,6 +222,25 @@ autoscaling: # Memory is a floor to provision under 'resources', not a signal to scale on. # targetMemoryUtilizationPercentage: 80 # behavior: {} + # Opt-in per-pod workload targets, rendered as autoscaling/v2 `Pods` metrics + # named `litellm_requests_per_second` and `litellm_tokens_per_second` with an + # AverageValue target, alongside whichever resource targets are set (the HPA + # follows the metric asking for the most replicas). A Prometheus Adapter must + # serve those two names on custom.metrics.k8s.io from the proxy's counters, + # grouped by the scrape target's `pod` label (enable serviceMonitor below so + # every pod is scraped on its own): + # litellm_requests_per_second: + # sum(rate(litellm_proxy_total_requests_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>) + # litellm_tokens_per_second: + # sum(rate(litellm_total_tokens_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>) + # rate() over [1m] is already per second, so no `* 60`. How fast the HPA + # reacts is set by that window, the scrape interval and the HPA sync period + # (15s by default), not by the unit: keep serviceMonitor.interval at 15s or + # faster so a 1m window holds at least 4 samples. averageValue takes SI + # suffixes, so "6M" is six million tokens per second per pod. Tokens are + # counted when a response completes, so TPS trails long streams. + targetRequestsPerSecond: "" + targetTokensPerSecond: "" # Autoscaling with keda is mutually exclusive with hpa keda: @@ -243,6 +262,23 @@ keda: # metricName: http_requests_total # threshold: '100' # query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) + # First-class Prometheus triggers on the proxy's own request and token + # counters, appended to `triggers`. Each target is the per-second load one + # replica should carry: KEDA divides the release-wide + # `sum(rate([1m]))` by it to pick the replica count. Thresholds + # are plain numbers (KEDA parses them as floats, no SI suffixes). The + # queries select samples by the release namespace and the `job` label the + # chart's ServiceMonitor produces (the metrics Service name), so enable + # serviceMonitor below together with metricsServer: the http port serves + # /metrics/ behind virtual-key auth and answers an unauthenticated scrape + # with 401. Reaction time comes from the [1m] window, the scrape interval + # and pollingInterval above, so keep both at 15s or faster. Tokens are + # counted at completion, so TPS trails long streams. serverAddress is + # required once either target is set. + prometheus: + serverAddress: "" + requestsPerSecond: "" + tokensPerSecond: "" behavior: {} # scaleDown: # stabilizationWindowSeconds: 300 diff --git a/helm/litellm/templates/gateway/hpa.yaml b/helm/litellm/templates/gateway/hpa.yaml index e97cef95ffb..4183079cd31 100644 --- a/helm/litellm/templates/gateway/hpa.yaml +++ b/helm/litellm/templates/gateway/hpa.yaml @@ -30,6 +30,24 @@ spec: type: Utilization averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }} {{- end }} + {{- with .Values.gateway.hpa.targetRequestsPerSecond }} + - type: Pods + pods: + metric: + name: litellm_requests_per_second + target: + type: AverageValue + averageValue: {{ toJson . | trimAll "\"" | quote }} + {{- end }} + {{- with .Values.gateway.hpa.targetTokensPerSecond }} + - type: Pods + pods: + metric: + name: litellm_tokens_per_second + target: + type: AverageValue + averageValue: {{ toJson . | trimAll "\"" | quote }} + {{- end }} {{- with .Values.gateway.hpa.behavior }} behavior: {{- toYaml . | nindent 4 }} diff --git a/helm/litellm/templates/gateway/servicemonitor.yaml b/helm/litellm/templates/gateway/servicemonitor.yaml new file mode 100644 index 00000000000..e1bafa6e388 --- /dev/null +++ b/helm/litellm/templates/gateway/servicemonitor.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.gateway.enabled .Values.gateway.serviceMonitor.enabled }} +{{- if not .Values.gateway.metricsServer.enabled }} +{{- fail "gateway.serviceMonitor.enabled requires gateway.metricsServer.enabled: the http port serves /metrics/ behind virtual-key auth, so an unauthenticated scrape gets 401" }} +{{- end }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "litellm.gateway.fullname" . }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: gateway + {{- with .Values.gateway.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "litellm.gateway.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace | quote }} + endpoints: + - port: metrics + path: /metrics/ + interval: {{ .Values.gateway.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.gateway.serviceMonitor.scrapeTimeout }} + scheme: http +{{- end }} diff --git a/helm/litellm/tests/hpa_workload_metrics_tests.yaml b/helm/litellm/tests/hpa_workload_metrics_tests.yaml new file mode 100644 index 00000000000..a29c53c5ed9 --- /dev/null +++ b/helm/litellm/tests/hpa_workload_metrics_tests.yaml @@ -0,0 +1,213 @@ +suite: test gateway HPA per-pod requests-per-second and tokens-per-second targets +templates: + - gateway/hpa.yaml + - gateway/servicemonitor.yaml +values: + - ./values/required.yaml +tests: + - it: scales on CPU and memory only by default + template: gateway/hpa.yaml + asserts: + - equal: + path: spec.metrics + value: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + + - it: adds a requests-per-second Pods metric next to the resource metrics + template: gateway/hpa.yaml + set: + gateway.hpa.targetRequestsPerSecond: 90 + asserts: + - lengthEqual: + path: spec.metrics + count: 3 + - contains: + path: spec.metrics + content: + type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + - equal: + path: spec.metrics[2] + value: + type: Pods + pods: + metric: + name: litellm_requests_per_second + target: + type: AverageValue + averageValue: "90" + - notContains: + path: spec.metrics + content: + type: Pods + pods: + metric: + name: litellm_tokens_per_second + any: true + + - it: adds a tokens-per-second Pods metric on its own + template: gateway/hpa.yaml + set: + gateway.hpa.targetTokensPerSecond: 6M + asserts: + - lengthEqual: + path: spec.metrics + count: 3 + - equal: + path: spec.metrics[2] + value: + type: Pods + pods: + metric: + name: litellm_tokens_per_second + target: + type: AverageValue + averageValue: "6M" + - notContains: + path: spec.metrics + content: + type: Pods + pods: + metric: + name: litellm_requests_per_second + any: true + + - it: renders requests and tokens targets together and keeps CPU and memory + template: gateway/hpa.yaml + set: + gateway.hpa.targetRequestsPerSecond: 90 + gateway.hpa.targetTokensPerSecond: 6000000 + asserts: + - lengthEqual: + path: spec.metrics + count: 4 + - equal: + path: spec.metrics[0].resource.name + value: cpu + - equal: + path: spec.metrics[1].resource.name + value: memory + - equal: + path: spec.metrics[2].pods.metric.name + value: litellm_requests_per_second + - equal: + path: spec.metrics[3].pods.metric.name + value: litellm_tokens_per_second + - equal: + path: spec.metrics[3].pods.target.averageValue + value: "6000000" + + - it: scales on workload metrics alone when the resource targets are cleared + template: gateway/hpa.yaml + set: + gateway.hpa.targetCPUUtilizationPercentage: null + gateway.hpa.targetMemoryUtilizationPercentage: null + gateway.hpa.targetRequestsPerSecond: 90 + gateway.hpa.targetTokensPerSecond: 6000000 + asserts: + - lengthEqual: + path: spec.metrics + count: 2 + - notContains: + path: spec.metrics + content: + type: Resource + any: true + - equal: + path: spec.metrics[0].pods.metric.name + value: litellm_requests_per_second + - equal: + path: spec.metrics[1].pods.metric.name + value: litellm_tokens_per_second + - notMatchRegexRaw: + pattern: per_minute + + - it: ignores the per-minute keys, which the chart never shipped + template: gateway/hpa.yaml + set: + gateway.hpa.targetRequestsPerMinute: 5400 + gateway.hpa.targetTokensPerMinute: 360000000 + asserts: + - lengthEqual: + path: spec.metrics + count: 2 + - notContains: + path: spec.metrics + content: + type: Pods + any: true + + - it: renders no ServiceMonitor by default + template: gateway/servicemonitor.yaml + asserts: + - hasDocuments: + count: 0 + + - it: refuses a ServiceMonitor without the metrics server, whose http port needs a bearer token + template: gateway/servicemonitor.yaml + set: + gateway.serviceMonitor.enabled: true + asserts: + - failedTemplate: + errorPattern: gateway.serviceMonitor.enabled requires gateway.metricsServer.enabled + + - it: scrapes each gateway pod through the metrics port + template: gateway/servicemonitor.yaml + release: + name: rel + namespace: llm + set: + gateway.serviceMonitor.enabled: true + gateway.metricsServer.enabled: true + gateway.serviceMonitor.labels: + release: kube-prometheus-stack + asserts: + - isKind: + of: ServiceMonitor + - equal: + path: metadata.labels.release + value: kube-prometheus-stack + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/name: litellm + app.kubernetes.io/instance: rel + app.kubernetes.io/component: gateway + - equal: + path: spec.namespaceSelector.matchNames + value: + - llm + - equal: + path: spec.endpoints + value: + - port: metrics + path: /metrics/ + interval: 15s + scrapeTimeout: 10s + scheme: http + + - it: honours a custom scrape interval + template: gateway/servicemonitor.yaml + set: + gateway.serviceMonitor.enabled: true + gateway.serviceMonitor.interval: 30s + gateway.metricsServer.enabled: true + asserts: + - equal: + path: spec.endpoints[0].interval + value: 30s diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index b5d535c992d..be1e6d43987 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -284,6 +284,16 @@ gateway: memory: 128Mi limits: memory: 512Mi + # Prometheus Operator ServiceMonitor for the gateway pods. Scrapes the + # `-metrics` Service, so it requires metricsServer above (the http + # port serves /metrics/ behind virtual-key auth). Every pod is its own scrape + # target, so the samples carry the `pod` label the per-pod autoscaling + # queries below group by. + serviceMonitor: + enabled: false + labels: {} + interval: 15s + scrapeTimeout: 10s image: repository: ghcr.io/berriai/litellm-gateway tag: "" # defaults to .Chart.AppVersion @@ -340,6 +350,25 @@ gateway: # policies: # - { type: Percent, value: 100, periodSeconds: 30 } behavior: {} + # Opt-in per-pod workload targets, rendered as autoscaling/v2 `Pods` metrics + # named `litellm_requests_per_second` and `litellm_tokens_per_second` with an + # AverageValue target. They coexist with the CPU/memory targets above: the + # HPA scales on whichever metric asks for the most replicas. Kubernetes has + # no idea what a token is, so a Prometheus Adapter must serve those two + # names on custom.metrics.k8s.io from the proxy's counters, grouped by the + # scrape target's `pod` label (enable serviceMonitor above): + # litellm_requests_per_second: + # sum(rate(litellm_proxy_total_requests_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>) + # litellm_tokens_per_second: + # sum(rate(litellm_total_tokens_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>) + # rate() over [1m] is already per second, so no `* 60`. How fast the HPA + # reacts is set by that window, the scrape interval and the HPA sync period + # (15s by default), not by the unit: keep serviceMonitor.interval at 15s or + # faster so a 1m window holds at least 4 samples. averageValue takes SI + # suffixes, so "6M" is six million tokens per second per pod. Tokens are + # counted when a response completes, so TPS trails long streams. + targetRequestsPerSecond: "" + targetTokensPerSecond: "" # PodDisruptionBudget for the gateway pods. Set exactly one of # `minAvailable` / `maxUnavailable` (minAvailable wins if both are set; # enabling without either falls back to `maxUnavailable: 1`). Disabled by diff --git a/terraform/litellm/aws/README.md b/terraform/litellm/aws/README.md index 923a4ca2da8..6d3e0269a15 100644 --- a/terraform/litellm/aws/README.md +++ b/terraform/litellm/aws/README.md @@ -258,6 +258,61 @@ gateway_metrics_port = 4001 gateway_metrics_scrape_cidrs = ["10.0.0.0/16"] ``` +### Scaling the gateway on requests and tokens + +By default the gateway service target-tracks CPU (`gateway_cpu_target`) and +memory (`gateway_memory_target`). Two more targets add workload signals next +to them. Application Auto Scaling evaluates every attached policy and follows +the one asking for the most tasks, so the resource policies keep working as a +floor while requests or tokens drive scale-out + +Both targets are per task per second, the way load is usually quoted (1k +rps, 75M tok/s). CloudWatch is the limit on how fast they react: target +tracking evaluates every metric, predefined or custom, aggregated over +60-second periods and has no period setting, so ECS reacts on a roughly +one-minute cadence whatever unit the variable is written in. The Kubernetes +charts get a faster signal because the Prometheus `rate()` window and scrape +interval are theirs to shorten + +`gateway_target_requests_per_second` adds an `ALBRequestCountPerTarget` +policy on the gateway target group. The ALB publishes that metric as requests +per minute per registered task, so the policy's target value is 60 times the +variable: 90 rps becomes a target of 5,400 per minute. No agent or sidecar is +needed + +`gateway_target_tokens_per_second` adds a metric-math policy over a +CloudWatch metric of the gateway's `litellm_total_tokens_metric_total` +counter and the service's `RunningTaskCount` from Container Insights. Nothing +native to ECS carries token throughput, so you publish that metric yourself +with the CloudWatch agent's Prometheus scraper pointed at the metrics sidecar +above. The agent emits the delta of a counter between scrapes, so `Sum` over +the 60-second period is the tokens served in that minute; the expression +divides by 60 (`tokens_per_second`) and then by the task count +(`tokens_per_second_per_task`). Tokens are counted when a response completes, +so long streams show up late in this signal. `gateway_tokens_metric` tells the +policy where the agent publishes: the namespace, the metric name (defaults to +the counter name) and the dimensions from your `metric_declaration` + +```hcl +gateway_metrics_port = 4001 +gateway_target_requests_per_second = 90 +gateway_target_tokens_per_second = 6000000 +gateway_tokens_metric = { + namespace = "LiteLLM/Prometheus" + dimensions = { ClusterName = "acme-litellm-prod", TaskDefinitionFamily = "acme-litellm-prod-gateway" } +} +``` + +Worked example for the request policy: 1,000 rps across 10 tasks is 100 rps +per task (the ALB reports it as 6,000 per minute per target) against a target +of 90 (5,400), so target tracking sizes the service to +`ceil(10 * 100 / 90) = 12` tasks. The token policy does the same arithmetic: +ten tasks handle 4,200,000,000 tokens in a minute, `tokens / 60` is +70,000,000 tokens per second and `tokens_per_second / running_tasks` is +7,000,000 against a target of 6,000,000, so the service grows to +`ceil(10 * 7000000 / 6000000) = 12`. Container Insights must be enabled on the +cluster for `RunningTaskCount` to exist + ## Tenant deployment Every resource the stack creates is named `${tenant}-litellm-${env}` (or diff --git a/terraform/litellm/aws/autoscaling.tf b/terraform/litellm/aws/autoscaling.tf index 71b6c24fac7..5197311d1b3 100644 --- a/terraform/litellm/aws/autoscaling.tf +++ b/terraform/litellm/aws/autoscaling.tf @@ -52,6 +52,105 @@ resource "aws_appautoscaling_policy" "gateway_memory" { } } +resource "aws_appautoscaling_policy" "gateway_requests" { + count = var.gateway_autoscaling_enabled && var.gateway_target_requests_per_second > 0 ? 1 : 0 + name = "${local.name}-gateway-requests" + policy_type = "TargetTrackingScaling" + service_namespace = aws_appautoscaling_target.gateway[0].service_namespace + resource_id = aws_appautoscaling_target.gateway[0].resource_id + scalable_dimension = aws_appautoscaling_target.gateway[0].scalable_dimension + + target_tracking_scaling_policy_configuration { + predefined_metric_specification { + predefined_metric_type = "ALBRequestCountPerTarget" + resource_label = "${aws_lb.this.arn_suffix}/${aws_lb_target_group.gateway.arn_suffix}" + } + # ALBRequestCountPerTarget is a per-minute count + target_value = var.gateway_target_requests_per_second * 60 + } +} + +resource "aws_appautoscaling_policy" "gateway_tokens" { + count = var.gateway_autoscaling_enabled && var.gateway_target_tokens_per_second > 0 ? 1 : 0 + name = "${local.name}-gateway-tokens" + policy_type = "TargetTrackingScaling" + service_namespace = aws_appautoscaling_target.gateway[0].service_namespace + resource_id = aws_appautoscaling_target.gateway[0].resource_id + scalable_dimension = aws_appautoscaling_target.gateway[0].scalable_dimension + + lifecycle { + precondition { + condition = var.gateway_tokens_metric != null + error_message = "gateway_tokens_metric is required when gateway_target_tokens_per_second > 0." + } + } + + target_tracking_scaling_policy_configuration { + target_value = var.gateway_target_tokens_per_second + + # target tracking has no period setting and always aggregates over 60s + customized_metric_specification { + metrics { + id = "tokens" + return_data = false + + metric_stat { + stat = "Sum" + + metric { + namespace = var.gateway_tokens_metric.namespace + metric_name = var.gateway_tokens_metric.name + + dynamic "dimensions" { + for_each = var.gateway_tokens_metric.dimensions + content { + name = dimensions.key + value = dimensions.value + } + } + } + } + } + + metrics { + id = "running_tasks" + return_data = false + + metric_stat { + stat = "Average" + + metric { + namespace = "ECS/ContainerInsights" + metric_name = "RunningTaskCount" + + dimensions { + name = "ClusterName" + value = aws_ecs_cluster.this.name + } + dimensions { + name = "ServiceName" + value = aws_ecs_service.gateway.name + } + } + } + } + + metrics { + id = "tokens_per_second" + expression = "tokens / 60" + return_data = false + } + + metrics { + id = "tokens_per_second_per_task" + expression = "tokens_per_second / running_tasks" + label = "Tokens per second per gateway task" + return_data = true + } + } + } +} + # ---------- Backend ---------- resource "aws_appautoscaling_target" "backend" { count = var.backend_autoscaling_enabled ? 1 : 0 diff --git a/terraform/litellm/aws/tests/workload_autoscaling.tftest.hcl b/terraform/litellm/aws/tests/workload_autoscaling.tftest.hcl new file mode 100644 index 00000000000..94e281faf94 --- /dev/null +++ b/terraform/litellm/aws/tests/workload_autoscaling.tftest.hcl @@ -0,0 +1,194 @@ +# Plan-only coverage for the gateway request and token autoscaling policies. +# Offline via mock_provider, same as byo_infrastructure.tftest.hcl. + +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } +} +mock_provider "random" {} + +variables { + region = "us-east-1" + tenant = "acme" + env = "test" + allow_plaintext_alb = true + azs = ["us-east-1a", "us-east-1b"] +} + +run "defaults_scale_on_cpu_and_memory_only" { + command = plan + + assert { + condition = alltrue([ + length(aws_appautoscaling_policy.gateway_cpu) == 1, + length(aws_appautoscaling_policy.gateway_memory) == 1, + length(aws_appautoscaling_policy.gateway_requests) == 0, + length(aws_appautoscaling_policy.gateway_tokens) == 0, + ]) + error_message = "Request and token policies must be absent by default while the CPU and memory policies stay." + } +} + +run "requests_per_second_adds_an_alb_request_count_policy" { + command = plan + + variables { + gateway_target_requests_per_second = 90 + } + + assert { + condition = length(aws_appautoscaling_policy.gateway_requests) == 1 && length(aws_appautoscaling_policy.gateway_tokens) == 0 + error_message = "A request target alone must add exactly the request policy." + } + + assert { + condition = alltrue([ + aws_appautoscaling_policy.gateway_requests[0].name == "acme-litellm-test-gateway-requests", + aws_appautoscaling_policy.gateway_requests[0].policy_type == "TargetTrackingScaling", + aws_appautoscaling_policy.gateway_requests[0].service_namespace == "ecs", + aws_appautoscaling_policy.gateway_requests[0].resource_id == "service/acme-litellm-test/acme-litellm-test-gateway", + aws_appautoscaling_policy.gateway_requests[0].scalable_dimension == "ecs:service:DesiredCount", + ]) + error_message = "The request policy must be a target-tracking policy on the gateway service's desired count." + } + + assert { + condition = alltrue([ + one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).target_value == 5400, + one(one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).predefined_metric_specification).predefined_metric_type == "ALBRequestCountPerTarget", + length(one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).customized_metric_specification) == 0, + ]) + error_message = "The request policy must track ALBRequestCountPerTarget at 60 times the configured requests per second per task." + } +} + +run "tokens_per_second_adds_a_metric_math_policy" { + command = plan + + variables { + gateway_target_tokens_per_second = 6000000 + gateway_tokens_metric = { + namespace = "LiteLLM/Prometheus" + dimensions = { ClusterName = "acme-litellm-test", TaskDefinitionFamily = "acme-litellm-test-gateway" } + } + } + + assert { + condition = length(aws_appautoscaling_policy.gateway_tokens) == 1 && length(aws_appautoscaling_policy.gateway_requests) == 0 + error_message = "A token target alone must add exactly the token policy." + } + + assert { + condition = alltrue([ + aws_appautoscaling_policy.gateway_tokens[0].name == "acme-litellm-test-gateway-tokens", + aws_appautoscaling_policy.gateway_tokens[0].resource_id == "service/acme-litellm-test/acme-litellm-test-gateway", + one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).target_value == 6000000, + length(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).predefined_metric_specification) == 0, + ]) + error_message = "The token policy must track a customized metric at the configured tokens per second per task." + } + + assert { + condition = alltrue([ + length(one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics) == 4, + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].id == "tokens", + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].return_data == false, + one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).stat == "Sum", + one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).namespace == "LiteLLM/Prometheus", + one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).metric_name == "litellm_total_tokens_metric_total", + { for d in one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).dimensions : d.name => d.value } == { ClusterName = "acme-litellm-test", TaskDefinitionFamily = "acme-litellm-test-gateway" }, + ]) + error_message = "The first metric must sum the published token counter deltas under the configured namespace and dimensions." + } + + assert { + condition = alltrue([ + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].id == "running_tasks", + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].return_data == false, + one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).stat == "Average", + one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).metric).namespace == "ECS/ContainerInsights", + one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).metric).metric_name == "RunningTaskCount", + { for d in one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).metric).dimensions : d.name => d.value } == { ClusterName = "acme-litellm-test", ServiceName = "acme-litellm-test-gateway" }, + ]) + error_message = "The second metric must read the gateway service's Container Insights RunningTaskCount." + } + + assert { + condition = alltrue([ + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second"].expression == "tokens / 60", + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second"].return_data == false, + ]) + error_message = "The 60s period Sum must be divided by 60 to yield tokens per second." + } + + assert { + condition = alltrue([ + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second_per_task"].expression == "tokens_per_second / running_tasks", + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second_per_task"].return_data == true, + length([for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m if m.return_data]) == 1, + ]) + error_message = "Only the per-task tokens per second may return data to the scaling policy." + } +} + +run "tokens_per_second_needs_the_metric_location" { + command = plan + + variables { + gateway_target_tokens_per_second = 6000000 + } + + expect_failures = [ + aws_appautoscaling_policy.gateway_tokens, + ] +} + +run "requests_and_tokens_scale_next_to_cpu_and_memory" { + command = plan + + variables { + gateway_target_requests_per_second = 90 + gateway_target_tokens_per_second = 6000000 + gateway_tokens_metric = { namespace = "LiteLLM/Prometheus" } + } + + assert { + condition = alltrue([ + length(aws_appautoscaling_policy.gateway_cpu) == 1, + length(aws_appautoscaling_policy.gateway_memory) == 1, + length(aws_appautoscaling_policy.gateway_requests) == 1, + length(aws_appautoscaling_policy.gateway_tokens) == 1, + one(aws_appautoscaling_policy.gateway_cpu[0].target_tracking_scaling_policy_configuration).target_value == 70, + one(aws_appautoscaling_policy.gateway_memory[0].target_tracking_scaling_policy_configuration).target_value == 80, + ]) + error_message = "Workload policies must coexist with the CPU and memory policies at their default targets." + } + + assert { + condition = length(one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).dimensions) == 0 + error_message = "Omitting dimensions must query the token metric without any." + } +} + +run "workload_targets_are_ignored_when_autoscaling_is_off" { + command = plan + + variables { + gateway_autoscaling_enabled = false + gateway_target_requests_per_second = 90 + gateway_target_tokens_per_second = 6000000 + gateway_tokens_metric = { namespace = "LiteLLM/Prometheus" } + } + + assert { + condition = alltrue([ + length(aws_appautoscaling_target.gateway) == 0, + length(aws_appautoscaling_policy.gateway_requests) == 0, + length(aws_appautoscaling_policy.gateway_tokens) == 0, + ]) + error_message = "Disabling gateway autoscaling must drop the workload policies with the target." + } +} diff --git a/terraform/litellm/aws/variables.tf b/terraform/litellm/aws/variables.tf index 667f6db63c9..10a9b392673 100644 --- a/terraform/litellm/aws/variables.tf +++ b/terraform/litellm/aws/variables.tf @@ -272,6 +272,47 @@ variable "gateway_memory_target" { default = 80 } +variable "gateway_target_requests_per_second" { + description = <<-EOT + Requests per second one gateway task should serve. Adds an + ALBRequestCountPerTarget target-tracking policy next to the CPU/memory + ones (Application Auto Scaling follows whichever asks for more tasks). + CloudWatch publishes that metric as a 1-minute count, so the policy + targets 60x this value and ECS reacts on a ~1 minute cadence. 0 skips + the policy. + EOT + type = number + default = 0 +} + +variable "gateway_target_tokens_per_second" { + description = <<-EOT + Tokens per second one gateway task should serve. Adds a target-tracking + policy on gateway_tokens_metric summed over each 60s period, divided by + 60 and by the service's Container Insights RunningTaskCount. Tokens are + counted when a response completes, so the signal trails long streams. + 0 skips the policy. + EOT + type = number + default = 0 +} + +variable "gateway_tokens_metric" { + description = <<-EOT + CloudWatch metric carrying the gateway's litellm_total_tokens_metric_total + counter, as published by the CloudWatch agent's Prometheus scraper (it + emits the delta between scrapes, so Sum over a period is the tokens + served in it). Required when gateway_target_tokens_per_second > 0. + dimensions must match the metric_declaration the agent publishes with. + EOT + type = object({ + namespace = string + name = optional(string, "litellm_total_tokens_metric_total") + dimensions = optional(map(string), {}) + }) + default = null +} + variable "backend_autoscaling_enabled" { description = "Toggle Application Auto Scaling target-tracking on the backend service." type = bool diff --git a/terraform/litellm/gcp/README.md b/terraform/litellm/gcp/README.md index c93e5f6b303..00ec7e8afd0 100644 --- a/terraform/litellm/gcp/README.md +++ b/terraform/litellm/gcp/README.md @@ -238,6 +238,24 @@ this with `litellm_license`. To tune the export cadence, set Behavior matches the AWS stack 1:1; the variable names are identical +### Autoscaling + +Cloud Run scales the gateway on request concurrency (plus its built-in CPU +target), not on a metric you attach. Each instance takes up to +`gateway_max_instance_request_concurrency` requests at once (default 80) +and Cloud Run adds instances between `gateway_min_instances` and +`gateway_max_instances` when the in-flight count fills up. That is the +request-rate signal for this stack: lower the concurrency for LLM streams +that hold a worker for tens of seconds, since a stream counts as one request +for as long as it is open + +There is no tokens-per-second path here. Cloud Run's autoscaler has no +custom-metric input, so the `litellm_total_tokens_metric_total` counter the +proxy exposes cannot drive it. If you need token-based scaling on GCP, run +the gateway on GKE with the Helm chart's `targetTokensPerSecond` (see +"Dependencies only" below) rather than wiring the counter into Cloud +Monitoring, which the autoscaler would ignore + ## Tenant deployment Every resource the stack creates is named `${tenant}-litellm-${env}` (or From 4de441c18100c39bdcca627b1051af40d9ff2849 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:41:37 -0700 Subject: [PATCH 062/126] test(ui): render TeamSSOSettings under a premium session so the organization dropdown tests fetch --- .../src/components/TeamSSOSettings.test.tsx | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx b/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx index 7b3153f6e13..4a042c32cda 100644 --- a/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx +++ b/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx @@ -8,6 +8,19 @@ import { toast } from "@/lib/toast"; vi.mock("./networking"); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ + token: "test-token", + accessToken: "test-token", + userId: "test-user", + userEmail: "test-user@example.com", + userRole: "Admin", + premiumUser: true, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }), +})); + vi.mock("./common_components/budget_duration_dropdown", () => { const BudgetDurationDropdown = ({ value, onChange }: { value: string | null; onChange: (value: string) => void }) => ( { onChange(event); @@ -232,9 +256,7 @@ const AddPluginForm: React.FC = ({ visible, onClose, accessT "Subfolder path (Optional)", "Path within the repository where the skill lives (e.g., plugins/my-skill). Leave empty if the skill is at the repo root.", )} - description={ - urlEncodesSubdir ? "The URL already points to a subfolder, so this field is disabled" : undefined - } + description={subPathLock ? SUB_PATH_LOCK_REASON[subPathLock] : undefined} > {({ ref, onChange, ...field }) => ( = ({ visible, onClose, accessT onChange(event); recomputePreview(form.getValues("skillUrl"), event.target.value); }} - disabled={urlEncodesSubdir} + disabled={subPathLock !== null} /> )} + {urlPreview?.parsed.source === "archive" && ( + + {({ ref, ...field }) => ( + + )} + + )} + {urlPreview && (
Detected: {urlPreview.label} diff --git a/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx index 2a1530cc352..1104e2c7485 100644 --- a/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx @@ -26,7 +26,7 @@ function getSkillSourceLink(skill: Plugin): { url: string; label: string } | nul const url = src.path ? `${src.url}/tree/main/${src.path}` : src.url; return { url, label: url.replace("https://github.com/", "") }; } - if (src?.source === "url" && src.url) { + if ((src?.source === "url" || src?.source === "archive") && src.url) { return { url: src.url, label: src.url.replace(/^https?:\/\//, "") }; } return null; diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts index 1d07ca09df5..4713fcbc55b 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts @@ -17,6 +17,7 @@ import { formatKeywords, parseSkillSource, isValidSubPath, + isValidSha256, buildMarketplaceSettingsSnippet, } from "./helpers"; import { MarketplacePluginEntry } from "./types"; @@ -114,6 +115,12 @@ describe("getSourceDisplayText", () => { ); }); + it("shows the archive url for an archive source", () => { + expect(getSourceDisplayText({ source: "archive", url: "https://bucket.s3.amazonaws.com/skill.zip" })).toBe( + "https://bucket.s3.amazonaws.com/skill.zip", + ); + }); + it("returns unknown for missing data", () => { expect(getSourceDisplayText({ source: "github" })).toBe("Unknown source"); }); @@ -140,6 +147,12 @@ describe("getSourceLink", () => { ); }); + it("returns the archive url for an archive source", () => { + expect(getSourceLink({ source: "archive", url: "https://bucket.s3.amazonaws.com/skill.zip" })).toBe( + "https://bucket.s3.amazonaws.com/skill.zip", + ); + }); + it("returns null when no repo or url", () => { expect(getSourceLink({ source: "github" })).toBeNull(); }); @@ -329,6 +342,20 @@ describe("isValidUrl", () => { }); }); +describe("isValidSha256", () => { + it("accepts an empty digest and a 64-character hex digest in either case", () => { + expect(isValidSha256("")).toBe(true); + expect(isValidSha256("a".repeat(64))).toBe(true); + expect(isValidSha256(" " + "ABCDEF0123456789".repeat(4) + " ")).toBe(true); + }); + + it("rejects wrong length and non-hex digests", () => { + expect(isValidSha256("a".repeat(63))).toBe(false); + expect(isValidSha256("a".repeat(65))).toBe(false); + expect(isValidSha256("g".repeat(64))).toBe(false); + }); +}); + describe("parseKeywords", () => { it("splits comma-separated keywords", () => { expect(parseKeywords("a, b, c")).toEqual(["a", "b", "c"]); @@ -445,6 +472,39 @@ describe("parseSkillSource", () => { expect(parseSkillSource("not a url")).toBeNull(); }); + it("parses an S3 zip URL into an archive source and names the skill after the file", () => { + expect(parseSkillSource("https://skills-bucket.s3.us-east-1.amazonaws.com/plugins/My_Skill-1.0.0.zip")).toEqual({ + parsed: { source: "archive", url: "https://skills-bucket.s3.us-east-1.amazonaws.com/plugins/My_Skill-1.0.0.zip" }, + label: "Zip archive — skills-bucket.s3.us-east-1.amazonaws.com/plugins/My_Skill-1.0.0.zip", + suggestedName: "my-skill-1-0-0", + }); + }); + + it("keeps the query string of a zip URL so versioned or signed object links still resolve", () => { + expect(parseSkillSource("https://bucket.s3.amazonaws.com/skill.ZIP?versionId=abc")?.parsed).toEqual({ + source: "archive", + url: "https://bucket.s3.amazonaws.com/skill.ZIP?versionId=abc", + }); + }); + + it("ignores the subfolder for a zip URL since the archive is installed whole", () => { + expect(parseSkillSource("https://artifacts.example.com/skill.zip", "plugins/x")?.parsed).toEqual({ + source: "archive", + url: "https://artifacts.example.com/skill.zip", + }); + }); + + it("rejects a plain http zip URL", () => { + expect(parseSkillSource("http://artifacts.example.com/skill.zip")).toBeNull(); + }); + + it("treats a github zip download URL as an archive rather than a repo path", () => { + expect(parseSkillSource("https://github.com/org/repo/releases/download/v1/skill.zip")?.parsed).toEqual({ + source: "archive", + url: "https://github.com/org/repo/releases/download/v1/skill.zip", + }); + }); + it("suggests a kebab-friendly name from the last path segment", () => { expect(parseSkillSource("github.com/org/my-awesome-skill")?.suggestedName).toBe("my-awesome-skill"); expect(parseSkillSource("github.com/org/repo/tree/main/plugins/cool-skill")?.suggestedName).toBe("cool-skill"); diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts index 5b3d4f20c1e..6f673d67f87 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts @@ -23,6 +23,12 @@ const GITHUB_HOST = "github.com"; const SKILL_FILE_EXTENSION_REGEX = /\.(md|markdown|txt|json|ya?ml|toml)$/i; +const ZIP_ARCHIVE_REGEX = /\.zip$/i; + +export const SHA256_REGEX = /^[0-9a-fA-F]{64}$/; + +export const isValidSha256 = (digest: string): boolean => digest.trim() === "" || SHA256_REGEX.test(digest.trim()); + // WHATWG normalizes obfuscated IPv4 (e.g. 2130706433, 0x7f.0.0.1) to dotted-decimal, so this // catches every IPv4 form; bracketed IPv6 is rejected separately. const IPV4_HOST_REGEX = /^\d{1,3}(\.\d{1,3}){3}$/; @@ -160,16 +166,26 @@ const parseRawGitSource = (url: URL, subPath?: string): SkillSourcePreview | nul }; }; +const parseArchiveSource = (url: URL): SkillSourcePreview => ({ + parsed: { source: "archive", url: url.href }, + label: `Zip archive — ${url.host}${url.pathname}`, + suggestedName: toKebabCase(lastSegment(url.pathname).replace(ZIP_ARCHIVE_REGEX, "")), +}); + /** - * Parse any git-accessible repository URL into a registerable skill source. - * GitHub URLs keep their `github`/`git-subdir` shorthand; every other host is - * treated as a raw repo URL, with an optional subfolder turning it into git-subdir. + * Parse any git-accessible repository URL or https zip archive URL into a registerable skill + * source. A `.zip` path is an `archive` source (S3, Artifactory, any static host). GitHub URLs + * keep their `github`/`git-subdir` shorthand; every other host is treated as a raw repo URL, + * with an optional subfolder turning it into git-subdir. */ export const parseSkillSource = (rawUrl: string, subPath?: string): SkillSourcePreview | null => { const url = parseRepoUrl(rawUrl); if (!url) { return null; } + if (ZIP_ARCHIVE_REGEX.test(url.pathname)) { + return parseArchiveSource(url); + } if (url.hostname.replace(/^www\./, "") === GITHUB_HOST) { return parseGitHubSource(url, subPath); } @@ -245,7 +261,7 @@ export const getSourceDisplayText = (source: PluginSource): string => { if (source.source === "git-subdir" && source.url && source.path) { return `${source.url} @ ${source.path}`; } - if (source.source === "url" && source.url) { + if ((source.source === "url" || source.source === "archive") && source.url) { return source.url; } return "Unknown source"; @@ -258,10 +274,8 @@ export const getSourceLink = (source: PluginSource): string | null => { if (source.source === "github" && source.repo) { return `https://github.com/${source.repo}`; } - if ((source.source === "url" || source.source === "git-subdir") && source.url) { - return source.url; - } - return null; + const linksToUrl = source.source === "url" || source.source === "git-subdir" || source.source === "archive"; + return linksToUrl && source.url ? source.url : null; }; /** diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx b/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx index 7cbbb08b623..b37204c6073 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx @@ -26,7 +26,7 @@ const SkillDetail: React.FC = ({ skill, onBack }) => { const src = skill.source; if (src.source === "github" && src.repo) return `https://github.com/${src.repo}`; if (src.source === "git-subdir" && src.url) return src.path ? `${src.url}/tree/main/${src.path}` : src.url; - if (src.source === "url" && src.url) return src.url; + if ((src.source === "url" || src.source === "archive") && src.url) return src.url; return null; })(); diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts index d16c880749b..98f80fd2959 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts @@ -8,10 +8,11 @@ import type { components } from "@/lib/http/schema"; // Kept hand-written: the backend types `source` as Dict[str, str], so the generated type is a // loose string map; this discriminant union is what the parser and display helpers rely on. export interface PluginSource { - source: "github" | "url" | "git-subdir"; + source: "github" | "url" | "git-subdir" | "archive"; repo?: string; // Format: "org/repo" for GitHub url?: string; // Full URL for other sources path?: string; // Subdirectory path for git-subdir + sha256?: string; } export type PluginAuthor = components["schemas"]["PluginAuthor"]; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index ab2c1f27a59..c7a760e1eee 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -2152,8 +2152,8 @@ export interface paths { * @description Register a new plugin in the LiteLLM marketplace. * * LiteLLM acts as a registry/discovery layer. Plugins are hosted on - * GitHub/GitLab/Bitbucket. Claude Code will clone from the git source - * when users install. + * GitHub/GitLab/Bitbucket or as a zip archive on any https host (e.g. S3). + * Claude Code clones the git source or downloads the archive when users install. * * This endpoint is create-only and never overwrites. If a plugin with * the same name already exists it returns 409 Conflict; use @@ -2163,7 +2163,7 @@ export interface paths { * * Parameters: * - name: Plugin name (kebab-case) - * - source: Git source reference (github, url, or git-subdir format) + * - source: Plugin source reference (github, url, git-subdir, or archive format) * - version: Semantic version (optional) * - description: Plugin description (optional) * - author: Author information (optional) @@ -2229,7 +2229,7 @@ export interface paths { * * Parameters: * - plugin_name: Name of the plugin to update (path parameter) - * - source: Git source reference (github, url, or git-subdir format) + * - source: Plugin source reference (github, url, git-subdir, or archive format) * - version: Semantic version (optional) * - description: Plugin description (optional) * - author: Author information (optional) @@ -33638,7 +33638,7 @@ export interface components { name: string; /** * Source - * @description Git source reference + * @description Plugin source reference */ source: { [key: string]: string; @@ -34887,7 +34887,7 @@ export interface components { * @description Request body for registering a plugin in the marketplace. * * LiteLLM acts as a registry/discovery layer. Plugins are hosted on - * GitHub/GitLab/Bitbucket and referenced by their git source. + * GitHub/GitLab/Bitbucket or as a zip archive on any https host and referenced by their source. */ RegisterPluginRequest: { /** @description Plugin author */ @@ -34929,10 +34929,11 @@ export interface components { namespace?: string | null; /** * Source - * @description Git source reference. Supported formats: + * @description Plugin source reference. Supported formats: * - GitHub: {'source': 'github', 'repo': 'org/repo'} * - Git URL: {'source': 'url', 'url': 'https://github.com/org/repo.git'} * - Git Subdir: {'source': 'git-subdir', 'url': 'https://github.com/org/repo.git', 'path': 'plugins/plugin-name'} + * - Zip archive on any https host (e.g. S3): {'source': 'archive', 'url': 'https://bucket.s3.amazonaws.com/plugin.zip', 'sha256': ''} */ source: { [key: string]: string; @@ -38211,10 +38212,11 @@ export interface components { namespace?: string | null; /** * Source - * @description Git source reference. Supported formats: + * @description Plugin source reference. Supported formats: * - GitHub: {'source': 'github', 'repo': 'org/repo'} * - Git URL: {'source': 'url', 'url': 'https://github.com/org/repo.git'} * - Git Subdir: {'source': 'git-subdir', 'url': 'https://github.com/org/repo.git', 'path': 'plugins/plugin-name'} + * - Zip archive on any https host (e.g. S3): {'source': 'archive', 'url': 'https://bucket.s3.amazonaws.com/plugin.zip', 'sha256': ''} */ source: { [key: string]: string; From 6cc13e07a68f1e822c1acbb7bec808c45baec8e2 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:26:09 -0700 Subject: [PATCH 078/126] feat(proxy): granular key/team access control for Claude Code marketplace plugins (#40518) Adds object_permission.skills to keys and teams, enforces it on /claude-code/marketplace.json?key=, /claude-code/plugins and /claude-code/plugins/{name}, and exposes an Allowed Skills selector in the key and team create/edit forms of the Admin UI Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../migration.sql | 1 + .../litellm_proxy_extras/schema.prisma | 1 + litellm/models/object_permission.py | 1 + litellm/proxy/_lazy_openapi_snapshot.json | 30 ++++- litellm/proxy/_types.py | 4 + .../claude_code_marketplace.py | 43 +++++-- .../claude_code_skill_access.py | 67 +++++++++++ litellm/proxy/schema.prisma | 1 + .../object_permission_repository.py | 6 + litellm/types/object_permission.py | 3 +- schema.prisma | 1 + .../test_claude_code_marketplace.py | 2 +- .../test_claude_code_marketplace.py | 97 +++++++++++++++- .../test_claude_code_skill_access.py | 79 +++++++++++++ .../proxy/auth/test_route_checks.py | 14 +++ .../test_customer_endpoints.py | 1 + .../test_object_permission_utils.py | 23 ++++ .../src/components/Teams.test.tsx | 29 +++++ ui/litellm-dashboard/src/components/Teams.tsx | 46 ++++++++ .../components/object_permissions_view.tsx | 11 ++ .../organisms/createKeyPayload.test.ts | 12 ++ .../components/organisms/createKeyPayload.ts | 5 + .../create_key_button.integration.test.tsx | 24 ++++ .../organisms/create_key_button.tsx | 31 +++++ .../components/skills/SkillSelector.test.tsx | 54 +++++++++ .../src/components/skills/SkillSelector.tsx | 109 ++++++++++++++++++ .../src/components/team/TeamInfo.test.tsx | 43 +++++++ .../src/components/team/TeamInfo.tsx | 26 +++++ .../templates/KeyEditViewControls.tsx | 34 +++++- .../KeyInfoView.handleKeyUpdate.test.tsx | 39 +++++++ .../components/templates/keyEditFormValues.ts | 4 + .../templates/key_edit_view.test.tsx | 40 +++++++ .../components/templates/key_edit_view.tsx | 15 +-- .../components/templates/key_info_view.tsx | 8 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 22 +++- 35 files changed, 892 insertions(+), 34 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260910000000_skills_on_object_permission/migration.sql create mode 100644 litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py create mode 100644 tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_skill_access.py create mode 100644 ui/litellm-dashboard/src/components/skills/SkillSelector.test.tsx create mode 100644 ui/litellm-dashboard/src/components/skills/SkillSelector.tsx diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260910000000_skills_on_object_permission/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260910000000_skills_on_object_permission/migration.sql new file mode 100644 index 00000000000..c982bc38a69 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260910000000_skills_on_object_permission/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN IF NOT EXISTS "skills" TEXT[] DEFAULT ARRAY[]::TEXT[]; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 05c5aad9303..817df082d8c 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -282,6 +282,7 @@ model LiteLLM_ObjectPermissionTable { mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user search_tools String[] @default([]) // search_tool_name values this key/team/user may call mcp_tool_search_enabled Boolean? + skills String[] @default([]) // Claude Code plugin names granted to this key/team beyond the public (enabled) set teams LiteLLM_TeamTable[] projects LiteLLM_ProjectTable[] verification_tokens LiteLLM_VerificationToken[] diff --git a/litellm/models/object_permission.py b/litellm/models/object_permission.py index a09d50ddc33..f178c5ad47f 100644 --- a/litellm/models/object_permission.py +++ b/litellm/models/object_permission.py @@ -23,3 +23,4 @@ class LiteLLM_ObjectPermissionTable(LiteLLMPydanticObjectBase): blocked_tools: list[str] | None = [] search_tools: list[str] | None = [] mcp_tool_search_enabled: bool | None = None + skills: list[str] | None = None diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 3f060f1f5f6..f0af17ab818 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -5721,8 +5721,26 @@ "paths": { "/claude-code/marketplace.json": { "get": { - "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add \n- claude plugin install @\n\nReturns:\n Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n ```bash\n claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n claude plugin install my-plugin@litellm\n ```", + "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add \n- claude plugin install @\n\nWithout `key` the catalog holds the enabled (public) plugins. With `?key=sk-...`\nthe key is authenticated and the catalog also holds the disabled plugins granted\nto it through `object_permission.skills` on the key or its team.\n\nReturns:\n Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n ```bash\n claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n claude plugin marketplace add \"http://localhost:4000/claude-code/marketplace.json?key=sk-...\"\n claude plugin install my-plugin@litellm\n ```", "operationId": "get_marketplace_claude_code_marketplace_json_get", + "parameters": [ + { + "in": "query", + "name": "key", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key" + } + } + ], "responses": { "200": { "content": { @@ -5731,6 +5749,16 @@ } }, "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" } }, "summary": "Get Marketplace", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f496140e905..23dc8237160 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -497,6 +497,9 @@ class LiteLLMRoutes(enum.Enum): "/v1/messages/count_tokens", "/v1/skills", "/v1/skills/{skill_id}", + "/claude-code/marketplace.json", + "/claude-code/plugins", + "/claude-code/plugins/{plugin_name}", ] # MCP tool-call / passthrough routes — data-plane. Gated by DISABLE_LLM_API_ENDPOINTS. @@ -1124,6 +1127,7 @@ class LiteLLM_ObjectPermissionBase(LiteLLMPydanticObjectBase): models: list[str] | None = None search_tools: list[str] | None = None mcp_tool_search_enabled: bool | None = None + skills: list[str] | None = None from litellm.models.team import BudgetLimitEntry as BudgetLimitEntry # noqa: E402 diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index cdeb1c6a111..7c6a4571948 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -7,10 +7,10 @@ Actual plugin files are hosted on GitHub/GitLab/Bitbucket or as a zip archive on any HTTPS host (S3, Artifactory, a static file server). Endpoints: -/claude-code/marketplace.json - GET - List plugins for Claude Code discovery (unauthenticated) +/claude-code/marketplace.json - GET - List plugins for Claude Code discovery (unauthenticated; `?key=` adds the key's granted skills) /claude-code/plugins - POST - Register a new plugin (create-only, proxy admin only) -/claude-code/plugins - GET - List plugins (any authenticated key) -/claude-code/plugins/{name} - GET - Get plugin details (any authenticated key) +/claude-code/plugins - GET - List plugins visible to the key (enabled, plus granted disabled ones) +/claude-code/plugins/{name} - GET - Get plugin details (403 on a disabled plugin the key is not granted) /claude-code/plugins/{name} - PUT - Update an existing plugin (proxy admin only) /claude-code/plugins/{name}/enable - POST - Enable a plugin (proxy admin only) /claude-code/plugins/{name}/disable - POST - Disable a plugin (proxy admin only) @@ -24,11 +24,15 @@ from datetime import datetime, timezone from typing import Annotated, Final, Protocol, TypedDict from urllib.parse import urlsplit -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import JSONResponse from litellm._logging import verbose_proxy_logger -from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth +from litellm.proxy._types import CommonProxyErrors, ProxyException, UserAPIKeyAuth +from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import ( + SkillVisibility, + skill_visibility, +) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.resource_ownership import is_proxy_admin from litellm.repositories.table_repositories import ClaudeCodePluginRepository @@ -84,7 +88,7 @@ async def _get_prisma_client() -> object: "/claude-code/marketplace.json", tags=["Claude Code Marketplace"], ) -async def get_marketplace(): +async def get_marketplace(request: Request, key: str | None = None): """ Serve marketplace.json for Claude Code plugin discovery. @@ -92,24 +96,35 @@ async def get_marketplace(): - claude plugin marketplace add - claude plugin install @ + Without `key` the catalog holds the enabled (public) plugins. With `?key=sk-...` + the key is authenticated and the catalog also holds the disabled plugins granted + to it through `object_permission.skills` on the key or its team. + Returns: Marketplace catalog with list of available plugins and their git sources. Example: ```bash claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json + claude plugin marketplace add "http://localhost:4000/claude-code/marketplace.json?key=sk-..." claude plugin install my-plugin@litellm ``` """ try: prisma_client: Final = await _get_prisma_client() + caller: Final[UserAPIKeyAuth | None] = ( + await user_api_key_auth(request=request, api_key=f"Bearer {key}") if key else None + ) + visibility: Final[SkillVisibility] = skill_visibility(caller) plugins: Final[Sequence[_PluginRecord]] = await ClaudeCodePluginRepository(prisma_client).table.find_many( - where={"enabled": True} + where=visibility.where() ) plugin_list: Final = [] for plugin in plugins: + if not visibility.allows(plugin): + continue try: manifest: Mapping[str, object] = json.loads(plugin.manifest_json or "{}") except json.JSONDecodeError: @@ -149,7 +164,7 @@ async def get_marketplace(): return JSONResponse(content=marketplace) - except HTTPException: + except (HTTPException, ProxyException): raise except Exception as e: verbose_proxy_logger.exception("Error generating marketplace: %s", e) @@ -395,13 +410,15 @@ async def list_plugins( try: prisma_client: Final = await _get_prisma_client() - where: Final = {"enabled": True} if enabled_only else {} + visibility: Final[SkillVisibility] = skill_visibility(user_api_key_dict) plugins: Final[Sequence[_PluginRecord]] = await ClaudeCodePluginRepository(prisma_client).table.find_many( - where=where + where={"enabled": True} if enabled_only else visibility.where() ) plugin_list: Final = [] for p in plugins: + if not visibility.allows(p): + continue # Parse manifest to get additional fields manifest = json.loads(p.manifest_json) if p.manifest_json else {} @@ -473,6 +490,12 @@ async def get_plugin( detail={"error": f"Plugin '{plugin_name}' not found"}, ) + if not skill_visibility(user_api_key_dict).allows(plugin): + raise HTTPException( + status_code=403, + detail={"error": f"Plugin '{plugin_name}' is not granted to this key"}, + ) + manifest: Final[Mapping[str, object]] = json.loads(plugin.manifest_json or "{}") if plugin.manifest_json else {} return { diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py new file mode 100644 index 00000000000..e1e1ee6f160 --- /dev/null +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py @@ -0,0 +1,67 @@ +""" +Claude Code marketplace visibility: enabled plugins are public, disabled plugins +are private and resolve only for proxy admins or keys granted them via +``object_permission.skills``. +""" + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Final, Protocol + +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth +from litellm.proxy.common_utils.resource_ownership import is_proxy_admin + +if TYPE_CHECKING: + from prisma.types import LiteLLM_ClaudeCodePluginTableWhereInput + + +class _SkillRecord(Protocol): + name: str + enabled: bool + + +def _skills_of(permission: LiteLLM_ObjectPermissionTable | None) -> frozenset[str]: + return frozenset(permission.skills or ()) if permission is not None else frozenset() + + +def granted_skills(user_api_key_dict: UserAPIKeyAuth) -> frozenset[str]: + """Key grant intersected with the team grant when both are non-empty; either alone applies as is. + + An empty list is the Prisma column default for every object-permission row, so it means + "no private grants configured here" and defers to the other scope, same as the agents check. + """ + key_skills: Final = _skills_of(user_api_key_dict.object_permission) + team_skills: Final = _skills_of(user_api_key_dict.team_object_permission) + match (bool(key_skills), bool(team_skills)): + case (True, True): + return key_skills & team_skills + case (True, False): + return key_skills + case _: + return team_skills + + +@dataclass(frozen=True, slots=True) +class SkillVisibility: + granted: frozenset[str] + sees_private: bool + + def allows(self, skill: _SkillRecord) -> bool: + return skill.enabled or self.sees_private or skill.name in self.granted + + def where(self) -> "LiteLLM_ClaudeCodePluginTableWhereInput": + if self.sees_private: + return {} + if not self.granted: + return {"enabled": True} + return {"OR": [{"enabled": True}, {"name": {"in": sorted(self.granted)}}]} + + +PUBLIC_ONLY: Final = SkillVisibility(granted=frozenset(), sees_private=False) + + +def skill_visibility(user_api_key_dict: UserAPIKeyAuth | None) -> SkillVisibility: + if user_api_key_dict is None: + return PUBLIC_ONLY + if is_proxy_admin(user_api_key_dict): + return SkillVisibility(granted=frozenset(), sees_private=True) + return SkillVisibility(granted=granted_skills(user_api_key_dict), sees_private=False) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 05c5aad9303..817df082d8c 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -282,6 +282,7 @@ model LiteLLM_ObjectPermissionTable { mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user search_tools String[] @default([]) // search_tool_name values this key/team/user may call mcp_tool_search_enabled Boolean? + skills String[] @default([]) // Claude Code plugin names granted to this key/team beyond the public (enabled) set teams LiteLLM_TeamTable[] projects LiteLLM_ProjectTable[] verification_tokens LiteLLM_VerificationToken[] diff --git a/litellm/repositories/object_permission_repository.py b/litellm/repositories/object_permission_repository.py index 6b1f9c68e47..7736939c696 100644 --- a/litellm/repositories/object_permission_repository.py +++ b/litellm/repositories/object_permission_repository.py @@ -40,6 +40,7 @@ class ObjectPermissionRepository(BaseRepository[LiteLLM_ObjectPermissionTable]): blocked_tools: list[str] | None = None, mcp_toolsets: list[str] | None = None, search_tools: list[str] | None = None, + skills: list[str] | None = None, ) -> LiteLLM_ObjectPermissionTable: """Create a new object permission record.""" data: Final[dict[str, Any]] = {} @@ -63,6 +64,8 @@ class ObjectPermissionRepository(BaseRepository[LiteLLM_ObjectPermissionTable]): data["mcp_toolsets"] = mcp_toolsets if search_tools is not None: data["search_tools"] = search_tools + if skills is not None: + data["skills"] = skills return await self.create(data) @@ -79,6 +82,7 @@ class ObjectPermissionRepository(BaseRepository[LiteLLM_ObjectPermissionTable]): blocked_tools: list[str] | None = None, mcp_toolsets: list[str] | None = None, search_tools: list[str] | None = None, + skills: list[str] | None = None, ) -> LiteLLM_ObjectPermissionTable | None: """Update an object permission record.""" data: Final[dict[str, Any]] = {} @@ -102,6 +106,8 @@ class ObjectPermissionRepository(BaseRepository[LiteLLM_ObjectPermissionTable]): data["mcp_toolsets"] = mcp_toolsets if search_tools is not None: data["search_tools"] = search_tools + if skills is not None: + data["skills"] = skills return await self.update(object_permission_id, data, id_field="object_permission_id") diff --git a/litellm/types/object_permission.py b/litellm/types/object_permission.py index 1b391a3a1ef..68661aed891 100644 --- a/litellm/types/object_permission.py +++ b/litellm/types/object_permission.py @@ -8,7 +8,7 @@ can adopt the type without violating the SDK-must-not-import-from-proxy layering rule. """ -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict class ObjectPermissionDict(TypedDict, total=False): @@ -23,3 +23,4 @@ class ObjectPermissionDict(TypedDict, total=False): models: list[str] | None search_tools: list[str] | None mcp_tool_search_enabled: bool | None + skills: ReadOnly[list[str] | None] diff --git a/schema.prisma b/schema.prisma index 05c5aad9303..817df082d8c 100644 --- a/schema.prisma +++ b/schema.prisma @@ -282,6 +282,7 @@ model LiteLLM_ObjectPermissionTable { mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user search_tools String[] @default([]) // search_tool_name values this key/team/user may call mcp_tool_search_enabled Boolean? + skills String[] @default([]) // Claude Code plugin names granted to this key/team beyond the public (enabled) set teams LiteLLM_TeamTable[] projects LiteLLM_ProjectTable[] verification_tokens LiteLLM_VerificationToken[] diff --git a/tests/pass_through_unit_tests/test_claude_code_marketplace.py b/tests/pass_through_unit_tests/test_claude_code_marketplace.py index 2ca81f1d5d3..bedb8830559 100644 --- a/tests/pass_through_unit_tests/test_claude_code_marketplace.py +++ b/tests/pass_through_unit_tests/test_claude_code_marketplace.py @@ -216,7 +216,7 @@ async def test_get_marketplace(mock_prisma_client): ) # Now get the marketplace - response = await get_marketplace() + response = await get_marketplace(request=MagicMock()) # Response is a JSONResponse, get the body body = json.loads(response.body.decode()) diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py index aa2da262bae..a7c2bd7ba20 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py @@ -11,17 +11,20 @@ from fastapi import HTTPException from unittest.mock import AsyncMock, MagicMock import litellm -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, ProxyException, UserAPIKeyAuth from litellm.proxy.proxy_server import LitellmUserRoles from litellm.types.proxy.claude_code_endpoints import ( RegisterPluginRequest, UpdatePluginRequest, ) +from litellm.proxy.anthropic_endpoints.claude_code_endpoints import claude_code_marketplace from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import ( delete_plugin, disable_plugin, enable_plugin, get_marketplace, + get_plugin, + list_plugins, register_plugin, update_plugin, ) @@ -38,11 +41,15 @@ def _make_mock_prisma(): async def _find_unique(where): return store.get(where.get("name")) + def _matches(record, where) -> bool: + if "OR" in where: + return any(_matches(record, clause) for clause in where["OR"]) + if "enabled" in where and record.enabled != where["enabled"]: + return False + return "name" not in where or record.name in where["name"]["in"] + async def _find_many(where=None): - records = list(store.values()) - if where and "enabled" in where: - return [r for r in records if r.enabled == where["enabled"]] - return records + return [r for r in store.values() if _matches(r, where or {})] async def _create(data): record = MagicMock() @@ -52,6 +59,8 @@ def _make_mock_prisma(): record.description = data.get("description") record.manifest_json = data.get("manifest_json", "{}") record.enabled = data.get("enabled", True) + record.created_at = data.get("created_at") + record.updated_at = data.get("updated_at") store[data["name"]] = record return record @@ -271,13 +280,89 @@ async def test_get_marketplace_skips_plugin_with_null_manifest(): table = litellm.proxy.proxy_server.prisma_client.db.litellm_claudecodeplugintable await table.create(data={"name": "null-manifest-plugin", "manifest_json": None, "enabled": True}) - response = await get_marketplace() + response = await get_marketplace(request=MagicMock()) assert response.status_code == 200 body = json.loads(response.body) assert [plugin["name"] for plugin in body["plugins"]] == ["good-plugin"] +def _granted_user(skills: list[str]) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + api_key="sk-granted", + user_id="granted-user", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="perm-1", skills=skills), + ) + + +async def _register_public_and_private_plugins() -> None: + for name, enabled in (("public-skill", True), ("private-skill", False)): + await register_plugin( + request=RegisterPluginRequest(name=name, source=_GIT_SUBDIR_SOURCE, version="1.0.0"), + user_api_key_dict=_USER, + ) + if not enabled: + await disable_plugin(plugin_name=name, user_api_key_dict=_USER) + + +async def _listed_names(user: UserAPIKeyAuth) -> set[str]: + response = await list_plugins(user_api_key_dict=user) + return {plugin.name for plugin in response.plugins} + + +@pytest.mark.asyncio +async def test_list_plugins_shows_disabled_plugin_only_to_granted_key_or_admin(): + await _register_public_and_private_plugins() + + assert await _listed_names(_NON_ADMIN_USER) == {"public-skill"} + assert await _listed_names(_granted_user(["other-skill"])) == {"public-skill"} + assert await _listed_names(_granted_user(["private-skill"])) == {"public-skill", "private-skill"} + assert await _listed_names(_USER) == {"public-skill", "private-skill"} + + +@pytest.mark.asyncio +async def test_get_plugin_returns_403_for_disabled_plugin_the_key_is_not_granted(): + await _register_public_and_private_plugins() + + with pytest.raises(HTTPException) as exc_info: + await get_plugin(plugin_name="private-skill", user_api_key_dict=_NON_ADMIN_USER) + assert exc_info.value.status_code == 403 + + assert (await get_plugin(plugin_name="public-skill", user_api_key_dict=_NON_ADMIN_USER))["name"] == "public-skill" + granted = await get_plugin(plugin_name="private-skill", user_api_key_dict=_granted_user(["private-skill"])) + assert granted["name"] == "private-skill" + assert granted["enabled"] is False + + +async def _marketplace_names(key: str | None) -> list[str]: + response = await get_marketplace(request=MagicMock(), key=key) + assert response.status_code == 200 + return sorted(plugin["name"] for plugin in json.loads(response.body)["plugins"]) + + +@pytest.mark.asyncio +async def test_get_marketplace_key_query_param_adds_granted_disabled_plugins(monkeypatch): + await _register_public_and_private_plugins() + keys = {"sk-granted": _granted_user(["private-skill"]), "sk-plain": _NON_ADMIN_USER} + + async def _fake_auth(request, api_key: str) -> UserAPIKeyAuth: + token = api_key.removeprefix("Bearer ") + if token not in keys: + raise ProxyException(message="invalid key", type="auth_error", param="key", code=401) + return keys[token] + + monkeypatch.setattr(claude_code_marketplace, "user_api_key_auth", _fake_auth) + + assert await _marketplace_names(None) == ["public-skill"] + assert await _marketplace_names("sk-plain") == ["public-skill"] + assert await _marketplace_names("sk-granted") == ["private-skill", "public-skill"] + + with pytest.raises(ProxyException) as exc_info: + await get_marketplace(request=MagicMock(), key="sk-bogus") + assert exc_info.value.code == "401" + + @pytest.mark.asyncio async def test_register_plugin_git_subdir_missing_url(): """git-subdir without url field raises HTTP 400.""" diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_skill_access.py b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_skill_access.py new file mode 100644 index 00000000000..dc4016d6db0 --- /dev/null +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_skill_access.py @@ -0,0 +1,79 @@ +"""Unit tests for claude_code_skill_access.py: key/team grant resolution.""" + +from unittest.mock import MagicMock + +import pytest + +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import ( + granted_skills, + skill_visibility, +) + + +def _perm(skills: list[str] | None) -> LiteLLM_ObjectPermissionTable: + return LiteLLM_ObjectPermissionTable(object_permission_id="perm", skills=skills) + + +def _key(key_skills: list[str] | None, team_skills: list[str] | None) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="sk-user", + user_role=LitellmUserRoles.INTERNAL_USER, + object_permission=_perm(key_skills) if key_skills is not None else None, + team_object_permission=_perm(team_skills) if team_skills is not None else None, + ) + + +def _plugin(name: str, enabled: bool) -> MagicMock: + record = MagicMock() + record.name = name + record.enabled = enabled + return record + + +@pytest.mark.parametrize( + ("key_skills", "team_skills", "expected"), + [ + (None, None, frozenset()), + ([], [], frozenset()), + (["a", "b"], None, frozenset({"a", "b"})), + (None, ["a", "b"], frozenset({"a", "b"})), + (["a", "b"], ["b", "c"], frozenset({"b"})), + (["a"], ["c"], frozenset()), + (["a", "b"], [], frozenset({"a", "b"})), + ([], ["a", "b"], frozenset({"a", "b"})), + ], +) +def test_granted_skills_intersects_key_with_team(key_skills, team_skills, expected): + assert granted_skills(_key(key_skills, team_skills)) == expected + + +def test_visibility_enabled_plugin_is_public_for_everyone(): + public = _plugin("public-skill", enabled=True) + + assert skill_visibility(None).allows(public) + assert skill_visibility(_key(None, None)).allows(public) + assert skill_visibility(_key([], ["other"])).allows(public) + + +def test_visibility_disabled_plugin_needs_grant_or_admin(): + private = _plugin("private-skill", enabled=False) + admin = UserAPIKeyAuth(api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN) + + assert not skill_visibility(None).allows(private) + assert not skill_visibility(_key(None, None)).allows(private) + assert not skill_visibility(_key(["other-skill"], None)).allows(private) + assert not skill_visibility(_key(["private-skill"], ["other-skill"])).allows(private) + assert skill_visibility(_key(["private-skill"], None)).allows(private) + assert skill_visibility(_key(None, ["private-skill"])).allows(private) + assert skill_visibility(admin).allows(private) + + +def test_where_clause_bounds_the_plugin_query_to_what_the_caller_may_see(): + admin = UserAPIKeyAuth(api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN) + + assert skill_visibility(None).where() == {"enabled": True} + assert skill_visibility(_key(None, None)).where() == {"enabled": True} + assert skill_visibility(_key(["b", "a"], None)).where() == {"OR": [{"enabled": True}, {"name": {"in": ["a", "b"]}}]} + assert skill_visibility(_key(["a", "b"], ["b"])).where() == {"OR": [{"enabled": True}, {"name": {"in": ["b"]}}]} + assert skill_visibility(admin).where() == {} diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 0fd19f518d9..4f2ff1582dd 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -3826,3 +3826,17 @@ def test_team_disable_logging_stays_proxy_admin_only(): def test_neighbouring_team_routes_stay_closed(route): """The grant is the callback paths and nothing else on the team namespace.""" assert "Only proxy admin" in _gate(route, LitellmUserRoles.INTERNAL_USER.value) + + +@pytest.mark.parametrize( + "route", + [ + "/claude-code/marketplace.json", + "/claude-code/plugins", + "/claude-code/plugins/my-skill", + ], +) +def test_claude_code_marketplace_routes_open_to_internal_users(route): + """Per-skill visibility is enforced inside the handler, so the route gate must let non-admins through.""" + assert RouteChecks.is_llm_api_route(route) is True + assert _gate(route, LitellmUserRoles.INTERNAL_USER.value) == "allowed" diff --git a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py index 1225cb80224..bd59a82cbd2 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py @@ -763,6 +763,7 @@ _EXPECTED_CUSTOMER = { "blocked_tools": [], "search_tools": [], "mcp_tool_search_enabled": None, + "skills": None, }, } diff --git a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py index d7ebb1f60bf..078315c2bf8 100644 --- a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py +++ b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py @@ -122,6 +122,29 @@ async def test_set_object_permission_persists_mcp_tool_search_enabled(): assert created_data["mcp_tool_search_enabled"] is True +@pytest.mark.asyncio +async def test_set_object_permission_persists_skills(): + mock_prisma_client = MagicMock() + mock_created_permission = MagicMock() + mock_created_permission.object_permission_id = "perm_id" + mock_prisma_client.db.litellm_objectpermissiontable.create = AsyncMock( + return_value=mock_created_permission + ) + + data_json = { + "object_permission": LiteLLM_ObjectPermissionBase(skills=["private-skill"]).model_dump(), + } + + await _set_object_permission(data_json=data_json, prisma_client=mock_prisma_client) + + created_data = ( + mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs[ + "data" + ] + ) + assert created_data["skills"] == ["private-skill"] + + # ---- Tests for _extract_requested_mcp_server_ids ---- diff --git a/ui/litellm-dashboard/src/components/Teams.test.tsx b/ui/litellm-dashboard/src/components/Teams.test.tsx index c7df35197e6..70c30596c75 100644 --- a/ui/litellm-dashboard/src/components/Teams.test.tsx +++ b/ui/litellm-dashboard/src/components/Teams.test.tsx @@ -33,6 +33,14 @@ vi.mock("./mcp_server_management/MCPServerSelector", () => ({ ), })); +vi.mock("./skills/SkillSelector", () => ({ + default: ({ onChange }: { onChange: (selected: string[]) => void }) => ( + + ), +})); + const can = vi.fn(); vi.mock("@/app/(dashboard)/hooks/useCan", () => ({ default: (...args: unknown[]) => can(...args), @@ -1359,6 +1367,27 @@ describe("Teams - the exact bytes the create call sends", () => { }); }); + it("puts the selected skills into object_permission.skills and drops the form key", async () => { + await openCreateModal(); + await openSection("Skill Settings", /Allowed Skills/); + fireEvent.click(screen.getByTestId("select-private-skill")); + + const payload = await submit(); + + expect(payload.object_permission).toStrictEqual({ skills: ["private-skill"] }); + expect(payload).not.toHaveProperty("object_permission_skills"); + }); + + it("sends no object_permission when Skill Settings is opened but nothing is selected", async () => { + await openCreateModal(); + await openSection("Skill Settings", /Allowed Skills/); + + const payload = await submit(); + + expect(payload).not.toHaveProperty("object_permission"); + expect(payload).not.toHaveProperty("object_permission_skills"); + }); + it("includes selected MCP toolsets in the create object permission", async () => { await openCreateModal(); await openSection("MCP Settings", /Allowed MCP Servers/); diff --git a/ui/litellm-dashboard/src/components/Teams.tsx b/ui/litellm-dashboard/src/components/Teams.tsx index c4060163c78..d2d18139893 100644 --- a/ui/litellm-dashboard/src/components/Teams.tsx +++ b/ui/litellm-dashboard/src/components/Teams.tsx @@ -50,6 +50,7 @@ import { Organization, getDefaultTeamSettings, getGuardrailsList, getPoliciesLis import NumericalInput from "./shared/numerical_input"; import VectorStoreSelector from "./vector_store_management/VectorStoreSelector"; import SearchToolSelector from "./search_tools/SearchToolSelector"; +import SkillSelector from "./skills/SkillSelector"; import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; interface TeamProps { @@ -99,6 +100,7 @@ const teamCreateFieldsSchema = z.object({ mcp_tool_permissions: z.record(z.string(), z.array(z.string())).optional(), allowed_agents_and_groups: z.object({ agents: z.array(z.string()), accessGroups: z.array(z.string()) }).optional(), object_permission_search_tools: z.array(z.string()).optional(), + object_permission_skills: z.array(z.string()).optional(), }); type TeamCreateFormValues = z.infer; @@ -128,6 +130,7 @@ const EMPTY_TEAM_CREATE_VALUES: TeamCreateFormValues = { mcp_tool_permissions: {}, allowed_agents_and_groups: undefined, object_permission_search_tools: undefined, + object_permission_skills: undefined, }; const ADDITIONAL_SETTINGS_FIELDS = [ @@ -147,6 +150,7 @@ const ADDITIONAL_SETTINGS_FIELDS = [ const MCP_SETTINGS_FIELDS = ["allowed_mcp_servers_and_groups", "mcp_tool_permissions"] as const; const AGENT_SETTINGS_FIELDS = ["allowed_agents_and_groups"] as const; const SEARCH_TOOL_SETTINGS_FIELDS = ["object_permission_search_tools"] as const; +const SKILL_SETTINGS_FIELDS = ["object_permission_skills"] as const; const isParsableJson = (value: string | undefined): boolean => { if (!value) { @@ -214,6 +218,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser const [mcpSettingsOpen, setMcpSettingsOpen] = useState(false); const [agentSettingsOpen, setAgentSettingsOpen] = useState(false); const [searchToolSettingsOpen, setSearchToolSettingsOpen] = useState(false); + const [skillSettingsOpen, setSkillSettingsOpen] = useState(false); const adminOrgs = useMemo( () => getAdminOrganizations(userRole, userID, organizations), @@ -505,6 +510,14 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser delete formValues.object_permission_search_tools; } + if (Array.isArray(formValues.object_permission_skills) && formValues.object_permission_skills.length > 0) { + if (!formValues.object_permission) { + formValues.object_permission = {}; + } + formValues.object_permission.skills = formValues.object_permission_skills; + } + delete formValues.object_permission_skills; + // Add model_aliases if any are defined if (Object.keys(modelAliases).length > 0) { formValues.model_aliases = modelAliases; @@ -540,6 +553,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser ...(mcpSettingsOpen ? [] : MCP_SETTINGS_FIELDS), ...(agentSettingsOpen ? [] : AGENT_SETTINGS_FIELDS), ...(searchToolSettingsOpen ? [] : SEARCH_TOOL_SETTINGS_FIELDS), + ...(skillSettingsOpen ? [] : SKILL_SETTINGS_FIELDS), ]); return Object.fromEntries(Object.entries(values).filter(([key]) => !unmounted.has(key))); }; @@ -1163,6 +1177,38 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser + + + Skill Settings + + + + + {({ value, onChange }) => ( + + )} + + + + Logging Settings diff --git a/ui/litellm-dashboard/src/components/object_permissions_view.tsx b/ui/litellm-dashboard/src/components/object_permissions_view.tsx index e90645a9e9f..ae46729222a 100644 --- a/ui/litellm-dashboard/src/components/object_permissions_view.tsx +++ b/ui/litellm-dashboard/src/components/object_permissions_view.tsx @@ -30,6 +30,7 @@ export function ObjectPermissionsView({ const agents = objectPermission?.agents || []; const agentAccessGroups = objectPermission?.agent_access_groups || []; const searchTools = objectPermission?.search_tools || []; + const skills = objectPermission?.skills || []; const content = (
@@ -58,6 +59,16 @@ export function ObjectPermissionsView({

{searchTools.join(", ")}

)}
+
+

Skills

+ {skills.length === 0 ? ( +

+ No private skills granted. Only enabled (public) Claude Code plugins are visible. +

+ ) : ( +

{skills.join(", ")}

+ )} +
); diff --git a/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts b/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts index 8b7283370a2..fef94cc3c2b 100644 --- a/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts +++ b/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts @@ -301,6 +301,16 @@ describe("object_permission", () => { ).toStrictEqual(aliasOnly({ object_permission: { agents: ["a-1"], agent_access_groups: ["ag-1"] } })); }); + it("moves the selected skills under object_permission and off the top level", () => { + expect(payloadOf(build({ key_alias: "my-key", allowed_skills: ["private-skill"] }))).toStrictEqual( + aliasOnly({ object_permission: { skills: ["private-skill"] } }), + ); + }); + + it("sends no object_permission for an empty skill selection", () => { + expect(payloadOf(build({ key_alias: "my-key", allowed_skills: [] }))).toStrictEqual(aliasOnly()); + }); + it("merges every source into a single object_permission", () => { const everySource = { key_alias: "my-key", @@ -308,6 +318,7 @@ describe("object_permission", () => { allowed_mcp_servers_and_groups: { servers: ["s-1"], accessGroups: ["g-1"], toolsets: ["t-1"] }, mcp_tool_permissions: { "s-1": ["read"] }, allowed_agents_and_groups: { agents: ["a-1"], accessGroups: ["ag-1"] }, + allowed_skills: ["private-skill"], }; expect(payloadOf(build(everySource))).toStrictEqual( aliasOnly({ @@ -319,6 +330,7 @@ describe("object_permission", () => { mcp_tool_permissions: { "s-1": ["read"] }, agents: ["a-1"], agent_access_groups: ["ag-1"], + skills: ["private-skill"], }, }), ); diff --git a/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts b/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts index a528dbaeb2c..37a973d5def 100644 --- a/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts +++ b/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts @@ -112,6 +112,7 @@ interface PermissionSources { readonly toolPermissions: unknown | undefined; readonly extraMcpAccessGroups: unknown[] | undefined; readonly agents: AgentSelection | undefined; + readonly skills: unknown[] | undefined; } const readPermissionSources = (values: Record): PermissionSources => ({ @@ -120,6 +121,7 @@ const readPermissionSources = (values: Record): PermissionSourc toolPermissions: readToolPermissions(values.mcp_tool_permissions), extraMcpAccessGroups: nonEmptyList(values.allowed_mcp_access_groups), agents: readAgentSelection(values.allowed_agents_and_groups), + skills: nonEmptyList(values.allowed_skills), }); const buildObjectPermission = ({ @@ -128,6 +130,7 @@ const buildObjectPermission = ({ toolPermissions, extraMcpAccessGroups, agents, + skills, }: PermissionSources): Record | undefined => { const permission: Record = { ...(vectorStores && { vector_stores: vectorStores }), @@ -138,6 +141,7 @@ const buildObjectPermission = ({ ...(extraMcpAccessGroups && { mcp_access_groups: extraMcpAccessGroups }), ...(agents?.agents && { agents: agents.agents }), ...(agents?.accessGroups && { agent_access_groups: agents.accessGroups }), + ...(skills && { skills }), }; return Object.keys(permission).length > 0 ? permission : undefined; }; @@ -148,6 +152,7 @@ const consumedSourceKeys = ( ): ReadonlySet => new Set([ "mcp_tool_permissions", + "allowed_skills", ...(values.disable_global_guardrails ? [] : ["disable_global_guardrails"]), ...(vectorStores ? ["allowed_vector_store_ids"] : []), ...(mcp ? ["allowed_mcp_servers_and_groups"] : []), diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx index 3471ef00eb0..2bf39bf4cd7 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx @@ -84,6 +84,13 @@ vi.mock("../networking", async (importOriginal) => { getPossibleUserRoles: vi.fn().mockResolvedValue({}), userFilterUICall: vi.fn().mockResolvedValue([]), getAgentsList: vi.fn().mockResolvedValue({ agents: [] }), + getClaudeCodePluginsList: vi.fn().mockResolvedValue({ + plugins: [ + { name: "public-skill", enabled: true }, + { name: "private-skill", enabled: false }, + ], + count: 2, + }), getPassThroughEndpointsCall: vi.fn().mockResolvedValue({ endpoints: [] }), vectorStoreListCall: vi.fn().mockResolvedValue({ data: [] }), listMCPTools: vi.fn().mockResolvedValue(emptyMcpTools), @@ -109,6 +116,7 @@ const OPENAPI_SCHEMA = { const SECTIONS = { mcp: /MCP Settings/i, agent: /Agent Settings/i, + skill: /Skill Settings/i, logging: /Logging Settings/i, router: /Router Settings/i, aliases: /Model Aliases/i, @@ -164,6 +172,7 @@ const ROUTER_SETTINGS_DEFAULT = { const SECTION_PAYLOAD_ADDITIONS: Record> = { mcp: { allowed_mcp_servers_and_groups: { servers: [], accessGroups: [] } }, agent: { allowed_agents_and_groups: undefined }, + skill: {}, logging: {}, router: { router_settings: ROUTER_SETTINGS_DEFAULT }, aliases: {}, @@ -329,6 +338,21 @@ describe("CreateKey", () => { expect(Object.keys(serialised).sort()).toStrictEqual([...wireKeys].sort()); }); + it("moves a picked private skill under object_permission.skills and off the top level", async () => { + await openModal(); + await nameTheKey(); + await openSection(/Optional Settings/i); + await openSection(SECTIONS.skill); + await userEvent.click(await screen.findByRole("combobox", { name: "Select skills (optional)" })); + await userEvent.click(await screen.findByRole("option", { name: "private-skill (private)" })); + await userEvent.keyboard("{Escape}"); + await submit(); + + const payload = await createdPayload(); + expect(payload.object_permission).toStrictEqual({ skills: ["private-skill"] }); + expect(payload).not.toHaveProperty("allowed_skills"); + }); + it("omits a budget typed into a section the user closed again, rather than sending it as null", async () => { await openModal(); await nameTheKey(); diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 16ea99ea94c..831cb0cf6a6 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -27,6 +27,7 @@ import React, { useEffect, useMemo, useRef, useState } from "react"; import { type Control, useForm, useWatch, type UseFormSetValue } from "react-hook-form"; import { rolesWithWriteAccess } from "../../utils/roles"; import AgentSelector from "../agent_management/AgentSelector"; +import SkillSelector from "../skills/SkillSelector"; import AccessGroupSelector from "../common_components/AccessGroupSelector"; import BudgetDurationDropdown from "../common_components/budget_duration_dropdown"; import SchemaFormFields from "../common_components/check_openapi_schema"; @@ -1557,6 +1558,36 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp + + + Skill Settings + + + + + Allowed Skills{" "} + + + + + } + name="allowed_skills" + help="Select private skills this key can access in the Claude Code marketplace" + > + {(control) => ( + + )} + + + + {premiumUser ? ( diff --git a/ui/litellm-dashboard/src/components/skills/SkillSelector.test.tsx b/ui/litellm-dashboard/src/components/skills/SkillSelector.test.tsx new file mode 100644 index 00000000000..59aff7c25de --- /dev/null +++ b/ui/litellm-dashboard/src/components/skills/SkillSelector.test.tsx @@ -0,0 +1,54 @@ +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import { getClaudeCodePluginsList } from "../networking"; +import SkillSelector from "./SkillSelector"; + +vi.mock("../networking", () => ({ + getClaudeCodePluginsList: vi.fn(), +})); + +describe("SkillSelector", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(getClaudeCodePluginsList).mockResolvedValue({ + plugins: [ + { name: "public-skill", enabled: true }, + { name: "private-skill", enabled: false }, + ], + count: 2, + }); + }); + + it("should list marketplace plugins and mark disabled ones as private", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("combobox")); + + expect(await screen.findByRole("option", { name: "public-skill" })).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "private-skill (private)" })).toBeInTheDocument(); + expect(getClaudeCodePluginsList).toHaveBeenCalledWith("token"); + }); + + it("should report the selected skill names", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + renderWithProviders(); + + await user.click(screen.getByRole("combobox")); + await user.click(await screen.findByRole("option", { name: "private-skill (private)" })); + + expect(onChange).toHaveBeenCalledWith(["private-skill"]); + }); + + it("should clear all selected skills", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: "Clear all skills" })); + + expect(onChange).toHaveBeenCalledWith([]); + }); +}); diff --git a/ui/litellm-dashboard/src/components/skills/SkillSelector.tsx b/ui/litellm-dashboard/src/components/skills/SkillSelector.tsx new file mode 100644 index 00000000000..b21c8b7e872 --- /dev/null +++ b/ui/litellm-dashboard/src/components/skills/SkillSelector.tsx @@ -0,0 +1,109 @@ +import React, { useEffect, useState } from "react"; +import { + Combobox, + ComboboxChip, + ComboboxChips, + ComboboxChipsInput, + ComboboxClear, + ComboboxContent, + ComboboxEmpty, + ComboboxItem, + ComboboxList, + ComboboxValue, + useComboboxAnchor, +} from "@/components/ui/combobox"; +import { cn } from "@/lib/cva.config"; +import { getClaudeCodePluginsList } from "../networking"; + +export interface SkillSelectorProps { + onChange: (selected: string[]) => void; + value?: string[]; + className?: string; + accessToken: string; + placeholder?: string; + disabled?: boolean; +} + +interface SkillOption { + readonly name: string; + readonly enabled: boolean; +} + +const readSkillOptions = (data: unknown): SkillOption[] => { + const plugins = (data as { plugins?: unknown[] } | undefined)?.plugins; + if (!Array.isArray(plugins)) return []; + return plugins.flatMap((plugin) => { + const record = plugin as { name?: unknown; enabled?: unknown }; + return typeof record.name === "string" && record.name.length > 0 + ? [{ name: record.name, enabled: record.enabled !== false }] + : []; + }); +}; + +const SkillSelector: React.FC = ({ + onChange, + value, + className, + accessToken, + placeholder = "Select skills (optional)", + disabled = false, +}) => { + const anchor = useComboboxAnchor(); + const [options, setOptions] = useState([]); + const [loading, setLoading] = useState(false); + + useEffect(() => { + const load = async () => { + if (!accessToken) return; + setLoading(true); + try { + setOptions(readSkillOptions(await getClaudeCodePluginsList(accessToken))); + } catch (e) { + console.error("Failed to load skills:", e); + } finally { + setLoading(false); + } + }; + load(); + }, [accessToken]); + + return ( + option.name)} + value={value ?? []} + onValueChange={(selected: string[]) => onChange(selected)} + disabled={disabled} + > + } className={cn("w-full", className)} aria-busy={loading}> + + {(selected: string[]) => + selected.map((skill) => ( + + {skill} + + )) + } + + + {value && value.length > 0 && } + + + {loading ? "Loading skills…" : "No skills found"} + + {(skill: string) => { + const isPrivate = options.some((option) => option.name === skill && !option.enabled); + return ( + + {skill} + {isPrivate && private} + + ); + }} + + + + ); +}; + +export default SkillSelector; diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index b286c8c1303..a25ca28651a 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -52,6 +52,7 @@ vi.mock("@/components/networking", () => ({ listMCPTools: vi.fn().mockResolvedValue({ tools: [] }), vectorStoreListCall: vi.fn().mockResolvedValue({ data: [] }), getAgentsList: vi.fn().mockResolvedValue({ agents: [] }), + getClaudeCodePluginsList: vi.fn().mockResolvedValue({ plugins: [], count: 0 }), })); const can = vi.fn(); @@ -1990,6 +1991,7 @@ describe("TeamInfoView - the exact bytes the update call sends", () => { agents: [], agent_access_groups: [], vector_stores: ["vs-1"], + skills: [], }; it("leaves every team member key out of the request body for an untouched save with both sections closed", async () => { @@ -2052,6 +2054,47 @@ describe("TeamInfoView - the exact bytes the update call sends", () => { expect(objectPermission.agent_access_groups).toStrictEqual([]); }); + it("resends the stored skills when the selector is left untouched", async () => { + const user = userEvent.setup({ delay: null }); + vi.mocked(networking.teamInfoCall).mockResolvedValue( + createMockTeamData({ models: ["gpt-4"], object_permission: { skills: ["private-skill"] } }), + ); + vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any); + + renderWithProviders(); + await waitFor(() => expect(screen.queryAllByText("Test Team").length).toBeGreaterThan(0)); + await user.click(screen.getByRole("tab", { name: "Settings" })); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + await screen.findByLabelText("Team Name"); + + const payload = await save(user); + + const objectPermission = wireBody(payload).object_permission as Record; + expect(objectPermission.skills).toStrictEqual(["private-skill"]); + }); + + it("sends an empty skills array after the last skill chip is removed", async () => { + const user = userEvent.setup({ delay: null }); + vi.mocked(networking.teamInfoCall).mockResolvedValue( + createMockTeamData({ models: ["gpt-4"], object_permission: { skills: ["private-skill"] } }), + ); + vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any); + + renderWithProviders(); + await waitFor(() => expect(screen.queryAllByText("Test Team").length).toBeGreaterThan(0)); + await user.click(screen.getByRole("tab", { name: "Settings" })); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + await screen.findByLabelText("Team Name"); + + await user.click(within(screen.getByLabelText("private-skill")).getByRole("button")); + expect(screen.queryByLabelText("private-skill")).not.toBeInTheDocument(); + + const payload = await save(user); + + const objectPermission = wireBody(payload).object_permission as Record; + expect(objectPermission.skills).toStrictEqual([]); + }); + it("sends an empty vector_stores array after the last vector store chip is removed", async () => { const user = userEvent.setup({ delay: null }); await openEditor(user); diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index 4947cec1441..988e2e082aa 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -89,6 +89,7 @@ import ObjectPermissionsView from "../object_permissions_view"; import NumericalInput from "../shared/numerical_input"; import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; import SearchToolSelector from "../search_tools/SearchToolSelector"; +import SkillSelector from "../skills/SkillSelector"; import EditLoggingSettings from "./EditLoggingSettings"; import RouterSettingsAccordion, { RouterSettingsAccordionRef } from "../common_components/RouterSettingsAccordion"; import MemberModal from "./EditMembership"; @@ -371,6 +372,7 @@ const teamUpdateFieldsSchema = z.object({ mcp_tool_permissions: z.record(z.string(), z.array(z.string())).optional(), agents_and_groups: z.object({ agents: z.array(z.string()), accessGroups: z.array(z.string()) }).optional(), object_permission_search_tools: z.array(z.string()).optional(), + object_permission_skills: z.array(z.string()).optional(), organization_id: z.string().nullish(), logging_settings: z.array(z.unknown()).optional(), secret_manager_settings: z.string().optional(), @@ -419,6 +421,7 @@ const EMPTY_TEAM_UPDATE_VALUES: TeamUpdateFormValues = { mcp_tool_permissions: {}, agents_and_groups: { agents: [], accessGroups: [] }, object_permission_search_tools: [], + object_permission_skills: [], organization_id: null, logging_settings: [], secret_manager_settings: "", @@ -485,6 +488,7 @@ const toTeamFormValues = (info: TeamInfoRecord, effectiveGuardrails: string[]): accessGroups: info.object_permission?.agent_access_groups || [], }, object_permission_search_tools: info.object_permission?.search_tools || [], + object_permission_skills: info.object_permission?.skills || [], organization_id: info.organization_id, logging_settings: info.metadata?.logging || [], secret_manager_settings: info.metadata?.secret_manager_settings @@ -1047,6 +1051,10 @@ const TeamInfoView: React.FC = ({ updateData.object_permission.search_tools = values.object_permission_search_tools; } + if (Array.isArray(values.object_permission_skills)) { + updateData.object_permission.skills = values.object_permission_skills; + } + // Pass access_group_ids to the update request if (values.access_group_ids !== undefined) { updateData.access_group_ids = values.access_group_ids; @@ -1848,6 +1856,24 @@ const TeamInfoView: React.FC = ({ + + {({ value, onChange }) => ( + + )} + + {({ id, value, onChange }) => ( ( <> @@ -53,6 +55,36 @@ export const KeyTypeSelect = ({ ); +const SKILLS_HINT = + "Enabled skills are visible to every key. Grant disabled (private) Claude Code plugins to this key here."; + +export const KeyAgentAndSkillFields = ({ + control, + accessToken, +}: { + control: Control; + accessToken: string; +}) => ( + <> + + {({ value, onChange }) => ( + + )} + + + + {({ value, onChange }) => ( + + )} + + +); + export const KeyBudgetNumberField = ({ control, name, diff --git a/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx b/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx index 8a42ecff50c..522af5a85ad 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx @@ -366,6 +366,45 @@ describe("KeyInfoView handleKeyUpdate mcp_toolsets", () => { }); }); +describe("KeyInfoView handleKeyUpdate skills", () => { + it("should forward the skills the edit form supplies into object_permission and drop the form key", async () => { + renderView(true); + + fireEvent.click(screen.getByText("Settings")); + fireEvent.click(screen.getByText("Edit Settings")); + (globalThis as any).__TEST_FORM_VALUES = { + token: "tok_123", + skills: ["private-skill"], + }; + + fireEvent.click(screen.getByText("Mock Submit")); + + await waitFor(() => expect(keyUpdateCallMock).toHaveBeenCalled()); + + const [, sentPayload] = keyUpdateCallMock.mock.calls[0]; + expect(sentPayload.object_permission.skills).toEqual(["private-skill"]); + expect(sentPayload).not.toHaveProperty("skills"); + }); + + it("should send an explicit empty skills list when the form clears every skill", async () => { + renderView(true); + + fireEvent.click(screen.getByText("Settings")); + fireEvent.click(screen.getByText("Edit Settings")); + (globalThis as any).__TEST_FORM_VALUES = { + token: "tok_123", + skills: [], + }; + + fireEvent.click(screen.getByText("Mock Submit")); + + await waitFor(() => expect(keyUpdateCallMock).toHaveBeenCalled()); + + const [, sentPayload] = keyUpdateCallMock.mock.calls[0]; + expect(sentPayload.object_permission.skills).toEqual([]); + }); +}); + describe("KeyInfoView handleKeyUpdate budget_duration", () => { it("should send a canonical budget_duration through unchanged", async () => { renderView(true); diff --git a/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts b/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts index f24fb6e5a86..fec8e749143 100644 --- a/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts +++ b/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts @@ -46,6 +46,7 @@ export interface KeyEditFormValues { mcp_servers_and_groups?: McpServersAndGroups; mcp_tool_permissions?: Record; agents_and_groups?: AgentsAndGroups; + skills?: string[]; organization_id?: string | null; team_id?: string | null; logging_settings?: unknown[]; @@ -102,6 +103,7 @@ export const toKeyEditFormValues = (keyData: KeyResponse): KeyEditFormValues => agents: keyData.object_permission?.agents || [], accessGroups: keyData.object_permission?.agent_access_groups || [], }, + skills: keyData.object_permission?.skills || [], organization_id: keyData.organization_id, team_id: keyData.team_id, logging_settings: extractLoggingSettings(keyData.metadata), @@ -148,6 +150,7 @@ export const keyEditFormSchema = z.object({ mcp_servers_and_groups: z.custom(), mcp_tool_permissions: z.custom | undefined>(), agents_and_groups: z.custom(), + skills: z.custom(), organization_id: z.custom(), team_id: z.custom(), logging_settings: z.custom(), @@ -196,6 +199,7 @@ export const toSubmittedValues = ( mcp_servers_and_groups: values.mcp_servers_and_groups, mcp_tool_permissions: values.mcp_tool_permissions, agents_and_groups: values.agents_and_groups, + skills: values.skills, organization_id: values.organization_id, team_id: values.team_id, logging_settings: values.logging_settings, diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx index 35e9268e1c2..b2c2a381b42 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx @@ -59,6 +59,7 @@ vi.mock("../networking", async () => { agents: [], }), getAgentAccessGroups: vi.fn().mockResolvedValue([]), + getClaudeCodePluginsList: vi.fn().mockResolvedValue({ plugins: [], count: 0 }), }; }); @@ -135,6 +136,14 @@ vi.mock("../agent_management/AgentSelector", () => ({ ), })); +vi.mock("../skills/SkillSelector", () => ({ + default: ({ onChange }: { onChange: (selected: string[]) => void }) => ( + + ), +})); + vi.mock("../common_components/AccessGroupSelector", () => ({ default: ({ value = [], onChange }: { value?: string[]; onChange?: (v: string[]) => void }) => ( { mcp_servers_and_groups: { servers: [], accessGroups: [], toolsets: [] }, mcp_tool_permissions: {}, agents_and_groups: { agents: [], accessGroups: [] }, + skills: [], organization_id: null, team_id: null, logging_settings: [], @@ -2241,6 +2251,36 @@ describe("KeyEditView", () => { expect(onSubmitMock.mock.calls[0][0].agents_and_groups.agents).toEqual(["agent-1"]); }); + it("carries a picked skill into the payload", async () => { + const onSubmitMock = vi.fn().mockResolvedValue(undefined); + renderForPayload(onSubmitMock); + await screen.findByRole("button", { name: /save changes/i }); + + await userEvent.click(screen.getByRole("button", { name: "pick skill" })); + await userEvent.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(onSubmitMock).toHaveBeenCalled(); + }); + expect(onSubmitMock.mock.calls[0][0].skills).toEqual(["private-skill"]); + }); + + it("preloads the stored skills into the payload when the selector is left untouched", async () => { + const onSubmitMock = vi.fn().mockResolvedValue(undefined); + renderForPayload(onSubmitMock, { + ...MOCK_KEY_DATA, + object_permission: { ...MOCK_KEY_DATA.object_permission, skills: ["stored-skill"] }, + } as KeyResponse); + await screen.findByRole("button", { name: /save changes/i }); + + await userEvent.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(onSubmitMock).toHaveBeenCalled(); + }); + expect(onSubmitMock.mock.calls[0][0].skills).toEqual(["stored-skill"]); + }); + it("carries an added logging integration into the payload", async () => { const onSubmitMock = vi.fn().mockResolvedValue(undefined); renderForPayload(onSubmitMock); diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index b1bbbeed6f9..e464dfe5008 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -15,7 +15,6 @@ import { FormField } from "@/components/shared/form/FormField"; import React, { useEffect, useRef, useState } from "react"; import { hasCapability } from "../../utils/capabilities"; import { isProxyAdminRole, rolesWithWriteAccess } from "../../utils/roles"; -import AgentSelector from "../agent_management/AgentSelector"; import AccessGroupSelector from "../common_components/AccessGroupSelector"; import BudgetDurationDropdown from "../common_components/budget_duration_dropdown"; import { mapInternalToDisplayNames } from "../callback_info_helpers"; @@ -32,9 +31,8 @@ import { modelSentinelOptions, parseAllowedRoutes, } from "./keyEditFieldNormalizers"; -import { KeyBudgetNumberField, KeyTypeSelect, labelWithHint } from "./KeyEditViewControls"; +import { KeyAgentAndSkillFields, KeyBudgetNumberField, KeyTypeSelect, labelWithHint } from "./KeyEditViewControls"; import { - AgentsAndGroups, KeyEditFormValues, keyEditFormSchema, McpServersAndGroups, @@ -762,16 +760,7 @@ export function KeyEditView({ /> - - {({ value, onChange }) => ( - - )} - + * - claude plugin install @ * + * Without `key` the catalog holds the enabled (public) plugins. With `?key=sk-...` + * the key is authenticated and the catalog also holds the disabled plugins granted + * to it through `object_permission.skills` on the key or its team. + * * Returns: * Marketplace catalog with list of available plugins and their git sources. * * Example: * ```bash * claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json + * claude plugin marketplace add "http://localhost:4000/claude-code/marketplace.json?key=sk-..." * claude plugin install my-plugin@litellm * ``` */ @@ -29333,6 +29338,8 @@ export interface components { models?: string[] | null; /** Search Tools */ search_tools?: string[] | null; + /** Skills */ + skills?: string[] | null; /** Vector Stores */ vector_stores?: string[] | null; }; @@ -29386,6 +29393,8 @@ export interface components { * @default [] */ search_tools: string[] | null; + /** Skills */ + skills?: string[] | null; /** * Vector Stores * @default [] @@ -43206,7 +43215,9 @@ export interface operations { }; get_marketplace_claude_code_marketplace_json_get: { parameters: { - query?: never; + query?: { + key?: string | null; + }; header?: never; path?: never; cookie?: never; @@ -43222,6 +43233,15 @@ export interface operations { "application/json": unknown; }; }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; }; }; list_plugins_claude_code_plugins_get: { From d62493a7794b1963c0c2ab541bda92d6dd896041 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:26:11 -0700 Subject: [PATCH 079/126] test(batch): prove an open Redis breaker keeps enqueued-token reservations quiet --- .../proxy/hooks/test_batch_enqueued_tokens.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_litellm/proxy/hooks/test_batch_enqueued_tokens.py b/tests/test_litellm/proxy/hooks/test_batch_enqueued_tokens.py index edc921a40a3..e3e39a87009 100644 --- a/tests/test_litellm/proxy/hooks/test_batch_enqueued_tokens.py +++ b/tests/test_litellm/proxy/hooks/test_batch_enqueued_tokens.py @@ -7,6 +7,7 @@ response-shape helpers the v3 limiter's post-call hooks rely on. """ import base64 +import logging import socket import uuid from collections.abc import Mapping, Sequence @@ -16,6 +17,7 @@ from typing import Final import pytest from litellm.caching.caching import DualCache +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.constants import BATCH_ENQUEUED_TOKEN_TTL_SECONDS from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.batch_enqueued_tokens import ( @@ -439,3 +441,29 @@ async def test_redis_lua_path_full_lifecycle(): refill = await store.reserve(tokens=50, scopes=(key_scope, team_scope)) assert isinstance(refill, BatchEnqueuedTokenReservation) await store.refund(refill) + + +class _OpenBreakerRedis: + def async_register_script(self, script: str): + async def refused(keys: Sequence[str], args: Sequence[str | bytes | int | float]) -> object: + raise RedisCircuitBreakerOpenError("Redis circuit breaker is open") + + return refused + + +@pytest.mark.asyncio +async def test_an_open_circuit_breaker_keeps_reservations_in_memory_without_a_warning(caplog): + scope = _scope(limit=100) + store = BatchEnqueuedTokenStore( + internal_usage_cache=InternalUsageCache(DualCache(redis_cache=_OpenBreakerRedis(), default_in_memory_ttl=60)) # pyright: ignore[reportArgumentType] # duck-typed Redis double + ) + + with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"): + reservation = await store.reserve(tokens=60, scopes=(scope,)) + assert isinstance(reservation, BatchEnqueuedTokenReservation) + assert reservation.backend == "memory" + await store.save_reservation("batch_quiet", reservation) + assert await store.pop_reservation("batch_quiet") == reservation + + assert not [record for record in caplog.records if record.levelno >= logging.WARNING] + assert sum("circuit breaker is open" in record.getMessage() for record in caplog.records) == 3 From c7d3a6a1d46d4e11c37df7caaa59f4fb582c4dea Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:31:48 -0700 Subject: [PATCH 080/126] refactor(redis): drop the narrative docstrings on the breaker helper and its tests --- litellm/caching/redis_cache.py | 7 +------ tests/test_litellm/caching/test_dual_cache.py | 5 ----- tests/test_litellm/caching/test_redis_cache.py | 5 ----- 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index d938b94df43..6b93529e456 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -393,17 +393,12 @@ def _record_swallowed_redis_failure(breaker: RedisCircuitBreaker, exc: BaseExcep class RedisCircuitBreakerOpenError(Exception): - """Raised in place of a Redis call while the circuit breaker is open.""" + pass def log_redis_failure( logger: logging.Logger, level: int, message: str, exc: BaseException, with_traceback: bool = False ) -> None: - """Log a Redis failure the caller is about to swallow. - - An open breaker refuses every call until Redis recovers and announced itself once when it - opened, so the calls it refuses are logged at debug instead of once per request at ``level``. - """ if isinstance(exc, RedisCircuitBreakerOpenError): logger.debug("%s: %s", message, exc) return diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index b0c17507adb..eae8bbfdaff 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -581,8 +581,6 @@ async def test_dual_cache_late_attach_redis_wires_writes_and_ttl_async(): class _OpenBreakerRedis: - """A RedisCache whose breaker is open, so every guarded call is refused before it starts.""" - def __init__(self) -> None: from litellm.caching.redis_cache import RedisCircuitBreaker @@ -639,9 +637,6 @@ class _OpenBreakerRedis: ids=["get", "batch_get", "set", "set_pipeline", "increment_pipeline", "increment"], ) async def test_an_open_circuit_breaker_is_not_an_error_per_request(caplog, call): - """While the breaker is open every request is refused by design, and the breaker already - said so once when it opened; logging each refusal as an ERROR traceback was the storm that - pinned every worker's CPU during a Redis latency blip.""" cache = DualCache(in_memory_cache=InMemoryCache(), redis_cache=_OpenBreakerRedis()) # pyright: ignore[reportArgumentType] # duck-typed Redis double caplog.clear() diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index b9d965a8942..bcaa58c9c40 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -1016,11 +1016,6 @@ async def test_breaker_metrics_track_state_and_failure_class(): def test_sync_guard_counts_a_timeout_as_a_timeout(): - """A sync Redis timeout must wait out the timeout-only min duration exactly like the async guard. - - Recording it as a hard connectivity failure opened the breaker on the fifth slow reply, - which is how a latency blip took the shared cache out for every worker. - """ from redis.exceptions import TimeoutError as RedisTimeoutError from litellm.caching.redis_cache import RedisCircuitBreaker, _run_under_circuit_breaker_sync From ad78a8f8d0a4e6200a98d5ef0ddae252b1e5c154 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 10 Sep 2026 21:35:49 +0000 Subject: [PATCH 081/126] refactor(caching): walk exception causes iteratively for the breaker timeout check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/redis_cache.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index de4dcb5aa27..742c7144784 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -15,7 +15,7 @@ import hashlib import inspect import json import time -from collections.abc import Awaitable, Callable, Sequence +from collections.abc import Awaitable, Callable, Iterator, Sequence from contextvars import ContextVar from dataclasses import dataclass from datetime import timedelta @@ -334,11 +334,22 @@ def _redis_timeout_error_types() -> tuple[type, ...]: return (RedisTimeoutError, TimeoutError) +_MAX_EXCEPTION_CAUSE_DEPTH: Final = 20 + + +def _exception_cause_chain(exc: BaseException) -> Iterator[BaseException]: + current = exc # rebind-ok: advances one link per iteration of the bounded walk + for _ in range(_MAX_EXCEPTION_CAUSE_DEPTH): + yield current + if current.__cause__ is None: + return + current = current.__cause__ + + def _is_redis_timeout_failure(exc: BaseException) -> bool: - """Follows __cause__: a blocking pool wait raises ConnectionError from asyncio.TimeoutError.""" - if isinstance(exc, _redis_timeout_error_types()): - return True - return exc.__cause__ is not None and _is_redis_timeout_failure(exc.__cause__) + """Walks __cause__: a blocking pool wait raises ConnectionError from asyncio.TimeoutError.""" + timeout_types: Final = _redis_timeout_error_types() + return any(isinstance(cause, timeout_types) for cause in _exception_cause_chain(exc)) class _BreakerMetrics: From 29145d13965df6aed5ebefa9e5ba8662321d54e8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:37:39 -0700 Subject: [PATCH 082/126] test(limiter): inject the open-breaker Redis double instead of replacing script attributes --- .../hooks/test_parallel_request_limiter_v3.py | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index f97d89b34a2..10c0bb88a82 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -6238,17 +6238,28 @@ async def test_post_call_success_hook_leaves_raw_provider_dict_untouched(): assert response == {"id": "msg_123", "type": "message", "role": "assistant", "content": []} -@pytest.mark.asyncio -async def test_an_open_circuit_breaker_falls_back_to_the_pipeline_without_a_warning(caplog): - from litellm.caching.redis_cache import RedisCircuitBreakerOpenError - from litellm.types.caching import RedisPipelineIncrementOperation +class _OpenBreakerRedis: + def async_register_script(self, script: str): + async def refused(keys, args): + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError - handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + raise RedisCircuitBreakerOpenError("Redis circuit breaker is open") + + return refused + + async def async_increment_pipeline(self, increment_list, **kwargs): + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError - async def refused_script(keys, args): raise RedisCircuitBreakerOpenError("Redis circuit breaker is open") - handler.token_increment_script = refused_script + +@pytest.mark.asyncio +async def test_an_open_circuit_breaker_falls_back_to_the_pipeline_without_a_warning(caplog): + from litellm.types.caching import RedisPipelineIncrementOperation + + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache(redis_cache=_OpenBreakerRedis())) # pyright: ignore[reportArgumentType] # duck-typed Redis double + ) with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"): await handler.async_increment_tokens_with_ttl_preservation( @@ -6261,14 +6272,9 @@ async def test_an_open_circuit_breaker_falls_back_to_the_pipeline_without_a_warn @pytest.mark.asyncio async def test_an_open_circuit_breaker_reads_the_sliding_window_locally_without_a_warning(caplog): - from litellm.caching.redis_cache import RedisCircuitBreakerOpenError - - handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) - - async def refused_script(keys, args): - raise RedisCircuitBreakerOpenError("Redis circuit breaker is open") - - handler.batch_rate_limiter_script = refused_script + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache(redis_cache=_OpenBreakerRedis())) # pyright: ignore[reportArgumentType] # duck-typed Redis double + ) with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"): values = await handler._execute_redis_batch_rate_limiter_script( From dbcb10c57d6546f232c31d1c0c73633edac449d0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:42:39 -0700 Subject: [PATCH 083/126] test(store_model_in_db): expect the placeholder model on rejected unknown-model spend logs The e2e suite asserted the pre-fix behavior (the raw client model string on the spend-log row). The fix in fe6f615c7a replaces it with unknown-model, so the assertion follows. --- tests/store_model_in_db_tests/test_openai_error_handling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/store_model_in_db_tests/test_openai_error_handling.py b/tests/store_model_in_db_tests/test_openai_error_handling.py index d3f38f93bf3..5de7c427d19 100644 --- a/tests/store_model_in_db_tests/test_openai_error_handling.py +++ b/tests/store_model_in_db_tests/test_openai_error_handling.py @@ -193,7 +193,7 @@ async def test_chat_completion_bad_model_with_spend_logs(): # Verify the structure of the log entry assert log_entry["request_id"] == litellm_call_id - assert log_entry["model"] == "non-existent-model" + assert log_entry["model"] == "unknown-model" assert log_entry["model_group"] in ("", "non-existent-model") assert log_entry["spend"] == 0.0 assert log_entry["total_tokens"] == 0 From df8a9c72ab44c9fa5552b1757f36d2fe99e39a7f Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 10 Sep 2026 21:45:46 +0000 Subject: [PATCH 084/126] fix(caching): keep the sync Redis read and the top-level cache wrapper quiet when the breaker is open The sync RedisCache.get_cache ran outside the breaker and logged with a stray positional argument, so every failure printed a logging-module stack dump. Cache.add_cache and its async twins logged a full traceback for the expected open-breaker fast fail Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/caching.py | 8 +++++- litellm/caching/dual_cache.py | 2 ++ litellm/caching/redis_cache.py | 4 ++- tests/test_litellm/caching/test_caching.py | 26 +++++++++++++++++++ tests/test_litellm/caching/test_dual_cache.py | 15 +++++++++++ .../test_litellm/caching/test_redis_cache.py | 23 +++++++++++++++- 6 files changed, 75 insertions(+), 3 deletions(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 884d095793c..b5423998401 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -32,7 +32,7 @@ from .dual_cache import DualCache # noqa: F401 from .gcs_cache import GCSCache from .in_memory_cache import InMemoryCache from .qdrant_semantic_cache import QdrantSemanticCache -from .redis_cache import RedisCache +from .redis_cache import RedisCache, RedisCircuitBreakerOpenError from .redis_cluster_cache import RedisClusterCache from .redis_semantic_cache import RedisSemanticCache from .s3_cache import S3Cache @@ -677,6 +677,8 @@ class Cache: return cache_key, cached_data, kwargs = self._add_cache_logic(result=result, **kwargs) self.cache.set_cache(cache_key, cached_data, **kwargs) + except RedisCircuitBreakerOpenError as e: + verbose_logger.debug("LiteLLM Cache: skipped add_cache: %s", e) except Exception as e: verbose_logger.exception("LiteLLM Cache: Excepton add_cache: %s", e) @@ -696,6 +698,8 @@ class Cache: await dynamic_cache_object.async_set_cache(cache_key, cached_data, **kwargs) else: await self.cache.async_set_cache(cache_key, cached_data, **kwargs) + except RedisCircuitBreakerOpenError as e: + verbose_logger.debug("LiteLLM Cache: skipped add_cache: %s", e) except Exception as e: verbose_logger.exception("LiteLLM Cache: Excepton add_cache: %s", e) @@ -875,6 +879,8 @@ class Cache: await dynamic_cache_object.async_set_cache_pipeline(cache_list=cache_list, **kwargs) else: await self.cache.async_set_cache_pipeline(cache_list=cache_list, **kwargs) + except RedisCircuitBreakerOpenError as e: + verbose_logger.debug("LiteLLM Cache: skipped add_cache: %s", e) except Exception as e: verbose_logger.exception("LiteLLM Cache: Excepton add_cache: %s", e) diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index 98c6da0f02f..6fb918ce748 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -177,6 +177,8 @@ class DualCache(BaseCache): print_verbose(f"get cache: cache result: {result}") return result + except RedisCircuitBreakerOpenError: + return None except Exception: verbose_logger.error(traceback.format_exc()) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 742c7144784..6c19b018c1d 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -1364,6 +1364,7 @@ class RedisCache(BaseCache): except Exception: return ast.literal_eval(decoded) + @_redis_circuit_breaker_guard_sync def get_cache(self, key, parent_otel_span: Span | None = None, **kwargs): try: key = self.check_and_fix_namespace(key=key) @@ -1384,7 +1385,8 @@ class RedisCache(BaseCache): return self._get_cache_logic(cached_response=cached_response) except Exception as e: # NON blocking - notify users Redis is throwing an exception - verbose_logger.error("litellm.caching.caching: get() - Got exception from REDIS: ", e) + verbose_logger.error("litellm.caching.caching: get() - Got exception from REDIS: %s", e) + _record_swallowed_redis_failure(self._circuit_breaker, e) def _run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]: """ diff --git a/tests/test_litellm/caching/test_caching.py b/tests/test_litellm/caching/test_caching.py index 4d0ec0fb677..5d326a9ece0 100644 --- a/tests/test_litellm/caching/test_caching.py +++ b/tests/test_litellm/caching/test_caching.py @@ -252,3 +252,29 @@ def test_exact_cache_key_includes_anthropic_messages_params(anthropic_param): assert baseline != cache.get_cache_key( model="claude-sonnet-4-5", messages=messages, **anthropic_param ) + + +@pytest.mark.asyncio +async def test_async_add_cache_treats_an_open_breaker_as_a_quiet_skip(caplog): + """The top-level write wrapper logs a full ERROR traceback for any failure. An open Redis + breaker fails every write instantly, so under load that wrapper alone was hundreds of + stack formats per second per replica. Unexpected failures must still get the traceback. + """ + from unittest.mock import AsyncMock + + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError + + caplog.set_level(logging.DEBUG, logger="LiteLLM") + cache = Cache(type=LiteLLMCacheType.LOCAL) + cache.cache.async_set_cache = AsyncMock(side_effect=RedisCircuitBreakerOpenError("open")) + + await cache.async_add_cache("result", model="gpt-5.4-nano", messages=[{"role": "user", "content": "hi"}]) + + assert [r.levelno for r in caplog.records if "add_cache" in r.getMessage()] == [logging.DEBUG] + cache.cache.async_set_cache.assert_awaited_once() + + cache.cache.async_set_cache = AsyncMock(side_effect=OSError("disk full")) + await cache.async_add_cache("result", model="gpt-5.4-nano", messages=[{"role": "user", "content": "hi"}]) + + errors = [r for r in caplog.records if r.levelno == logging.ERROR] + assert len(errors) == 1 and errors[0].exc_info is not None diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index 46dd2687e48..3e21362b3f0 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -638,3 +638,18 @@ async def test_open_breaker_is_a_quiet_cache_miss(dual_cache_with_open_breaker, noisy = [r for r in caplog.records if r.levelno > logging.DEBUG] assert noisy == [], f"an open breaker must be silent per call, got {[r.getMessage() for r in noisy]}" + + +def test_open_breaker_is_a_quiet_cache_miss_on_the_sync_read_path(dual_cache_with_open_breaker, caplog): + """The sync read runs in the request thread pool for /v1/messages and /v1/responses, so it + must short-circuit on an open breaker like the async path instead of dialing Redis per call. + """ + caplog.set_level(logging.DEBUG, logger="LiteLLM") + redis_client = dual_cache_with_open_breaker.redis_cache.redis_client + redis_client.get.side_effect = AssertionError("an open breaker must not touch Redis") + + for _ in range(50): + assert dual_cache_with_open_breaker.get_cache("lit7468") is None + + noisy = [r for r in caplog.records if r.levelno > logging.DEBUG] + assert noisy == [], f"an open breaker must be silent per call, got {[r.getMessage() for r in noisy]}" diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index 856f9b49c56..1a884d44db1 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -1,4 +1,5 @@ import asyncio +import logging from collections.abc import Iterator from unittest.mock import AsyncMock, MagicMock, patch @@ -646,7 +647,6 @@ def test_sync_batch_get_cache_survives_a_service_callback_that_raises( from concurrent.futures import ThreadPoolExecutor import litellm - from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD cache, service_logger = sync_batch_cache_with_service_logger @@ -1141,3 +1141,24 @@ async def test_recovery_probe_still_closes_the_breaker(): await asyncio.sleep(0.06) assert await _run_under_circuit_breaker(breaker, "probe", recovered) == "ok" assert breaker.is_open() is False + + +def test_sync_get_cache_failure_feeds_the_breaker_and_logs_a_well_formed_record(sync_batch_redis_cache, caplog): + """The sync read used to log with a stray positional arg, so every Redis failure produced a + `--- Logging error ---` stack dump on stderr, and it sat outside the breaker so it kept dialing + Redis on every request even after the async paths had opened it. + """ + from redis.exceptions import ConnectionError as RedisConnectionError + + from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD + + caplog.set_level(logging.ERROR, logger="LiteLLM") + sync_batch_redis_cache.redis_client.get.side_effect = RedisConnectionError("refused") + + for _ in range(REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD): + assert sync_batch_redis_cache.get_cache("lit7468") is None + + assert sync_batch_redis_cache._circuit_breaker.is_open() is True + assert sync_batch_redis_cache.redis_client.get.call_count == REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD + assert all("refused" in record.getMessage() for record in caplog.records) + assert len(caplog.records) == REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD From 1987e6e290a8344515f595538e85c3ae00e69a96 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:54:46 -0700 Subject: [PATCH 085/126] test(proxy): pass the request to get_marketplace in the archive marketplace test --- .../proxy/anthropic_endpoints/test_claude_code_marketplace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py index a7c2bd7ba20..a585666743f 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py @@ -481,7 +481,7 @@ async def test_archive_source_registers_and_is_served_verbatim_in_marketplace(): assert response.action == "created" assert response.plugin.source == _ARCHIVE_SOURCE - marketplace = json.loads((await get_marketplace()).body) + marketplace = json.loads((await get_marketplace(request=MagicMock())).body) assert marketplace["plugins"] == [{"name": "s3-skill", "source": _ARCHIVE_SOURCE, "version": "1.0.0"}] From 6632e8b74fecaeb569abdd12be43f6870b45c3b5 Mon Sep 17 00:00:00 2001 From: Kishorekarthik P Date: Sat, 1 Aug 2026 12:57:22 +0530 Subject: [PATCH 086/126] fix(proxy): let internal users read request/response for their own spend logs The Logs drawer gets messages/response from GET /spend/logs/ui/{request_id}; the list endpoint omits those heavy columns for every caller, admins included. That detail route was missing from LiteLLMRoutes.spend_tracking_routes, and check_route_access anchors patterns, so /spend/logs/ui never matched it. Every internal_user got a 403 before the handler ran and the UI fell back to the "Request/Response Data Not Available" banner, even on their own requests Adds the route to spend_tracking_routes so internal_user, internal_user_view_only, admin_viewer and org_admin all inherit it, and drops the now-redundant explicit entry from admin_viewer_routes. The handler already authorizes non-admins per row via _assert_user_can_view_request_id, so no handler-side scoping change is needed That helper returned silently when no spend-log row existed, which the detail handler treats as authorized before asking every custom logger for the payload by raw request_id. With retention pruning the row can be gone while the payload is still in cold storage, so opening the route would have let a non-admin read another tenant's prompt out of S3/GCS. A missing row now falls through to the same 403 as a foreign row, which also removes the exists-but-not-yours oracle Fixes #34099 --- litellm/proxy/_types.py | 9 +- .../spend_management_endpoints.py | 12 +- .../proxy/auth/test_route_checks.py | 76 +++++++ .../test_spend_management_endpoints.py | 186 ++++++++++++++++++ 4 files changed, 273 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 23dc8237160..aa585120e5b 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -703,6 +703,7 @@ class LiteLLMRoutes(enum.Enum): "/spend/logs", "/spend/logs/v2", "/spend/logs/ui", + "/spend/logs/ui/{request_id}", "/spend/logs/session/ui", "/key/spend/report", "/user/spend/report", @@ -932,10 +933,10 @@ class LiteLLMRoutes(enum.Enum): # PROXY_ADMIN_VIEW_ONLY — the route gate must match). "/customer/list", "/customer/info", - # UI Logs page detail drawer (single + session) and the filter facets. - # The list endpoint `/spend/logs/ui` is covered via - # spend_tracking_routes below. - "/spend/logs/ui/{logId}", + # UI Logs page session detail drawer and the end-user filter facet. + # The list endpoint `/spend/logs/ui` and the single-log detail route + # `/spend/logs/ui/{request_id}` are covered via spend_tracking_routes + # below. "/spend/logs/session/ui", "/management/v1/spend_logs/end_users", "/management/v1/spend_logs/users", diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index f8831ca4152..88cdf37cabb 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -4633,16 +4633,16 @@ async def _assert_user_can_view_request_id( Verify the requesting non-admin user is allowed to view this spend-log row. Allowed when the log belongs to the user directly, or to one of their permitted teams (admin or ``/spend/logs`` permission). - Raises HTTP 403 if not. + Raises HTTP 403 if not, including when no spend-log row exists for the + request_id (e.g. it was pruned by retention), so a missing row can't be + used to read a payload out of cold storage via the detail endpoint. """ row: Final = await _find_spend_log_row(prisma_client, request_id) - if row is None: + + if row is not None and row.user is not None and row.user == user_api_key_dict.user_id: return - if row.user is not None and row.user == user_api_key_dict.user_id: - return - - if row.team_id: + if row is not None and row.team_id: can_view: Final = await _can_team_member_view_log( prisma_client=prisma_client, user_api_key_dict=user_api_key_dict, diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 4f2ff1582dd..38a0e85c1ea 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -2033,6 +2033,82 @@ def test_proxy_admin_viewer_can_access_logs_page_endpoints(route): ) +@pytest.mark.parametrize( + "user_role", + [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY], +) +def test_internal_user_can_access_logs_drawer_detail_route(user_role): + """ + The Logs drawer detail fetch (GET /spend/logs/ui/{request_id}) must pass + route_checks for plain internal users, not just admins — the handler + itself already self-authorizes row ownership via + _assert_user_can_view_request_id. + """ + route = "/spend/logs/ui/abc-request-id" + user_obj = LiteLLM_UserTable( + user_id="internal_user", + user_email="user@example.com", + user_role=user_role.value, + ) + valid_token = UserAPIKeyAuth( + user_id="internal_user", + user_role=user_role.value, + ) + request = MagicMock(spec=Request) + request.query_params = {} + + try: + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=user_role.value, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + except Exception as e: + pytest.fail(f"{user_role.value} should be able to access {route}. Got error: {str(e)}") + + +@pytest.mark.parametrize( + "route_group_name", + [ + "spend_tracking_routes", + "internal_user_routes", + "internal_user_view_only_routes", + "admin_viewer_routes", + "org_admin_allowed_routes", + ], +) +def test_logs_drawer_detail_route_in_every_route_group(route_group_name): + """ + /spend/logs/ui/{request_id} must be reachable through + RouteChecks.check_route_access under each role's own route group, so a + partial revert (removing the route from `spend_tracking_routes` while + leaving `non_proxy_admin_allowed_routes_check` alone) is also caught. + """ + from litellm.proxy._types import LiteLLMRoutes + + allowed_routes = getattr(LiteLLMRoutes, route_group_name).value + assert RouteChecks.check_route_access( + route="/spend/logs/ui/req-34099", allowed_routes=allowed_routes + ) + + +def test_logs_drawer_detail_route_allowed_for_scoped_virtual_key(): + """ + A virtual key scoped to `allowed_routes=["spend_tracking_routes"]` must be + able to reach the Logs drawer detail route. + """ + valid_token = UserAPIKeyAuth( + user_id="scoped_key_user", + allowed_routes=["spend_tracking_routes"], + ) + assert RouteChecks.is_virtual_key_allowed_to_call_route( + route="/spend/logs/ui/req-34099", valid_token=valid_token + ) + + @pytest.mark.parametrize("route", ADMIN_VIEWER_LOGS_PAGE_ROUTES) def test_internal_user_blocked_from_admin_viewer_logs_routes(route): """ diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 329a33eb440..53accfd08ee 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -455,6 +455,34 @@ async def test_assert_user_can_view_request_id_rejects_both_users_none(): assert exc_info.value.status_code == 403 +@pytest.mark.asyncio +async def test_assert_user_can_view_request_id_rejects_missing_row(): + """ + A request_id with no spend-log row (e.g. pruned by retention) must not + authorize reading the payload from cold storage; a missing row is not + the same as an owned row. + """ + + class MockSpendLogs: + async def find_unique(self, where, include=None): + return None + + class MockDB: + def __init__(self): + self.litellm_spendlogs = MockSpendLogs() + + class MockPrisma: + def __init__(self): + self.db = MockDB() + + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="user_1") + with pytest.raises(HTTPException) as exc_info: + await spend_management_endpoints._assert_user_can_view_request_id( + MockPrisma(), auth, "req-missing-row" + ) + assert exc_info.value.status_code == 403 + + def test_ui_view_request_response_forbids_non_admin_without_db(client, monkeypatch): """ Without prisma, non-admins cannot be authorized to read request/response @@ -6777,3 +6805,161 @@ async def test_ui_view_spend_logs_search_returns_flat_rows_when_grouping_by_sess assert "next_session_cursor" not in data finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def _fake_prisma_with_owned_spend_log(owner_user_id, messages_json, response_json): + class _Row: + user = owner_user_id + team_id = None + + class _SpendLogs: + async def find_unique(self, where, include=None): + return _Row() + + class _DB: + def __init__(self): + self.litellm_spendlogs = _SpendLogs() + + async def query_raw(self, _sql, *_args): + return [ + { + "messages": messages_json, + "response": response_json, + "proxy_server_request": "{}", + "metadata": "{}", + } + ] + + class _Prisma: + def __init__(self): + self.db = _DB() + + return _Prisma() + + +def test_ui_view_request_response_internal_user_owner_gets_payload(client, monkeypatch): + """ + An internal_user who owns the spend-log row can fetch the Logs drawer + detail payload for their own request (regression for #34099, where the + route was blocked for INTERNAL_USER before reaching this ownership check). + """ + messages_json = json.dumps([{"role": "user", "content": "hi"}]) + response_json = json.dumps({"choices": [{"message": {"content": "hello"}}]}) + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + _fake_prisma_with_owned_spend_log("user_a", messages_json, response_json), + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="user_a" + ) + try: + response = client.get( + "/spend/logs/ui/req-owned-by-user-a", + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200 + body = response.json() + assert json.loads(body["messages"]) == [{"role": "user", "content": "hi"}] + assert json.loads(body["response"]) == { + "choices": [{"message": {"content": "hello"}}] + } + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +class _RecordingAdditionalLoggingUtils: + """Injectable custom logger that records every request_id it's asked for.""" + + def __init__(self, payload): + self._payload = payload + self.requested_ids = [] + + async def get_request_response_payload(self, request_id, start_time_utc, end_time_utc): + self.requested_ids.append(request_id) + return self._payload + + +def test_ui_view_request_response_internal_user_non_owner_forbidden(client, monkeypatch): + """ + A different internal_user requesting someone else's row is forbidden; + guards against _assert_user_can_view_request_id being skipped in the + detail-drawer handler. Also proves the handler stops before it ever asks + a custom logger or the DB for the payload. + """ + messages_json = json.dumps([{"role": "user", "content": "hi"}]) + response_json = json.dumps({"choices": [{"message": {"content": "hello"}}]}) + fake_prisma = _fake_prisma_with_owned_spend_log("user_a", messages_json, response_json) + original_query_raw = fake_prisma.db.query_raw + query_raw_calls = [] + + async def _spy_query_raw(*args, **kwargs): + query_raw_calls.append((args, kwargs)) + return await original_query_raw(*args, **kwargs) + + fake_prisma.db.query_raw = _spy_query_raw + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", fake_prisma) + + custom_logger = _RecordingAdditionalLoggingUtils({"messages": "should-not-be-returned"}) + monkeypatch.setattr( + litellm.logging_callback_manager, + "get_active_additional_logging_utils_from_custom_logger", + lambda: [custom_logger], + ) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="user_b" + ) + try: + response = client.get( + "/spend/logs/ui/req-owned-by-user-a", + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 403 + assert custom_logger.requested_ids == [] + assert query_raw_calls == [] + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_ui_view_request_response_internal_user_missing_row_forbidden(client, monkeypatch): + """ + Regression for the fail-open in _assert_user_can_view_request_id: a + request_id with no spend-log row (e.g. pruned by retention) must be + denied before the handler ever consults a custom logger, otherwise a + non-admin who guesses/obtains a request_id could read another tenant's + payload out of cold storage. Fails if `if row is None: return` is + reintroduced. + """ + + class _SpendLogs: + async def find_unique(self, where, include=None): + return None + + class _DB: + def __init__(self): + self.litellm_spendlogs = _SpendLogs() + + from types import SimpleNamespace + + fake_prisma = SimpleNamespace(db=_DB()) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", fake_prisma) + + custom_logger = _RecordingAdditionalLoggingUtils({"messages": "should-not-be-returned"}) + monkeypatch.setattr( + litellm.logging_callback_manager, + "get_active_additional_logging_utils_from_custom_logger", + lambda: [custom_logger], + ) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="user_a" + ) + try: + response = client.get( + "/spend/logs/ui/req-pruned", + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 403 + assert custom_logger.requested_ids == [] + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) From d98522b6f662fec6f600ab94817855cb7ff7edce Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 22:14:37 +0000 Subject: [PATCH 087/126] feat(proxy): share database connections across workers with an in-container pgbouncer (#39683) * feat(proxy): share database connections across workers with an in-container pgbouncer Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): parse pgbouncer options iteratively to satisfy the recursion gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): refuse pgbouncer with token db auth and retry failed pooler restarts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): build pgbouncer 1.25.2 from a pinned source archive and verify pooler replacements The public Wolfi repository only carries pgbouncer 1.24.1-r3, which the image scan rejects (CVE-2026-6664, CVE-2026-6665, CVE-2026-6666, CVE-2025-12819). All three images now compile the checksummed 1.25.2 release in a builder stage. The supervisor now waits for a replacement pooler to listen before treating it as recovered, ends and retries one that never does, and takes the same lock for stop() and spawn so no replacement can be started after shutdown began. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): refuse to start pgbouncer on a loopback port another process already owns Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): count pgbouncer ready only once its own unix socket answers, not any listener on the port Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): refuse pgbouncer older than 1.19, whose unix socket cannot vouch for the tcp port Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(helm,terraform): expose the in-container pgbouncer pool for the componentized gateway Add database.connectionPool to helm/litellm and gateway_connection_pool_* to terraform/litellm/aws so the componentized gateway can receive the LITELLM_PGBOUNCER_* env the classic image already honours. Both reject the pool under IAM or Entra token auth at render/plan time: the pooler holds one static database password for the life of the pod or task. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(gateway): launch the componentized gateway image through a pgbouncer-aware supervisor (#40592) The componentized gateway image started uvicorn directly, so the in-container PgBouncer never ran for it: every worker opened its own Prisma pool to the database. It also passed no keep-alive timeout, so behind a load balancer with a 60s idle timeout uvicorn's 5s default closed idle connections first and the balancer returned 502s on scale-out gateway.launch assembles DATABASE_URL, starts PgBouncer once per pod when LITELLM_PGBOUNCER_ENABLED is set, hands the workers the loopback URL and then runs uvicorn on gateway.main:app with KEEPALIVE_TIMEOUT as --timeout-keep-alive. The image builds PgBouncer 1.25.2 from a checksummed tarball, copies the compiled Rust extension into the /app source tree it imports from (it was only in site-packages, which PYTHONPATH=/app shadows) and asserts the native bridge loads. The app user is added to stats_users so operators can read the PgBouncer console with the application credentials The supervisor returns the pooled URL instead of writing into the mapping it was handed, a database user whose name PgBouncer would split into several stats_users entries is refused before the config is written, and the launcher tests drive main() with an injected serve callable Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * docs(terraform): describe the gateway.launch pooler entrypoint in the aws module README Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): run pgbouncer exit hooks only in the parent and copy the CA into the runtime dir Gunicorn workers inherit the parent's atexit table, so a recycled worker (max_requests) stopped the shared pooler and removed its runtime dir, then hung in the inherited Popen lock. The hooks now no-op unless os.getpid() is the process that started PgBouncer A verified TLS upstream named the operator's CA bundle directly, which is often a 0600 root-owned file that nobody (the user PgBouncer drops to) cannot read, so every server connection failed with "failed to load CA". The bundle is copied into the runtime dir next to the ini and chowned with it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(terraform): run the gateway through gateway.launch and add the gcp connection-pool variables Cloud Run and ECS overrode the image command with uvicorn gateway.main:app, which skips the supervisor that starts the in-container PgBouncer, so LITELLM_PGBOUNCER_ENABLED was inert on both stacks. Both now exec python -m gateway.launch (under ddtrace-run when USE_DDTRACE is set), and the gcp module gains gateway_connection_pool_enabled / gateway_pool_max_db_connections / gateway_pool_max_client_conn wired to the gateway service only The test_launch password_env fixture now restores DATABASE_URL even when it was unset: monkeypatch.delenv records nothing for an absent var, so main() left postgresql://...@db.internal in the xdist worker's environ and the key-rotation e2e test in the same proxy-infra shard stopped skipping and tried to reach db.internal Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(pgbouncer): keep channel_binding and gssencmode off the loopback URL Prisma would demand TLS channel binding from a pooler that only speaks plain TCP on 127.0.0.1. Also pass the request the marketplace test started needing after #40518 landed on top of #40496 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-unit.yml | 1 + Dockerfile | 19 +- docker/Dockerfile.database | 19 +- docker/Dockerfile.non_root | 19 +- gateway/Dockerfile | 28 +- gateway/launch.py | 71 ++ helm/litellm-helm/templates/deployment.yaml | 8 + .../tests/connection_pool_tests.yaml | 61 ++ helm/litellm-helm/values.yaml | 14 + helm/litellm/templates/_helpers.tpl | 17 + .../litellm/templates/gateway/deployment.yaml | 3 + helm/litellm/tests/connection_pool_tests.yaml | 117 ++++ helm/litellm/values.yaml | 20 + litellm/proxy/db/pgbouncer.py | 525 +++++++++++++++ litellm/proxy/proxy_cli.py | 16 + terraform/litellm/aws/README.md | 33 + terraform/litellm/aws/ecs.tf | 14 +- .../aws/tests/connection_pool.tftest.hcl | 123 ++++ terraform/litellm/aws/variables.tf | 39 ++ terraform/litellm/gcp/README.md | 34 + terraform/litellm/gcp/cloudrun.tf | 10 +- .../gcp/tests/connection_pool.tftest.hcl | 135 ++++ terraform/litellm/gcp/variables.tf | 39 ++ tests/test_gateway/test_launch.py | 155 +++++ tests/test_litellm/proxy/db/test_pgbouncer.py | 624 ++++++++++++++++++ .../test_litellm/test_component_entrypoint.py | 72 +- 26 files changed, 2195 insertions(+), 21 deletions(-) create mode 100644 gateway/launch.py create mode 100644 helm/litellm-helm/tests/connection_pool_tests.yaml create mode 100644 helm/litellm/tests/connection_pool_tests.yaml create mode 100644 litellm/proxy/db/pgbouncer.py create mode 100644 terraform/litellm/aws/tests/connection_pool.tftest.hcl create mode 100644 terraform/litellm/gcp/tests/connection_pool.tftest.hcl create mode 100644 tests/test_gateway/test_launch.py create mode 100644 tests/test_litellm/proxy/db/test_pgbouncer.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index cc606339a20..f55c87c2ae5 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -200,6 +200,7 @@ jobs: tests/test_litellm/proxy/types_utils tests/test_litellm/proxy/logging_endpoints tests/test_litellm/proxy/test_*.py + tests/test_gateway workers: 4 reruns: 2 timeout-minutes: 20 diff --git a/Dockerfile b/Dockerfile index 0a92aa9a68c..759dac76795 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,9 +8,25 @@ ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 +# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) +ARG PGBOUNCER_VERSION=1.25.2 +ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332 FROM $UV_IMAGE AS uvbin +FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder +ARG PGBOUNCER_VERSION +ARG PGBOUNCER_SHA256 +USER root +RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl +WORKDIR /build +RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \ + echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \ + tar xzf pgbouncer.tar.gz --strip-components=1 && \ + ./configure --prefix=/usr/local --with-openssl=/usr && \ + make -j"$(nproc)" pgbouncer && \ + install -m 0755 pgbouncer /usr/local/bin/pgbouncer + # Admin UI builder. Pinned to the build platform so the architecture-independent # Next.js static export compiles once natively even in a multi-arch build, # instead of once per target arch under QEMU. @@ -110,7 +126,8 @@ USER root RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories # node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile +RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent +COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer WORKDIR /app ENV PATH="/app/.venv/bin:${PATH}" \ diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index e9ad2849bb2..b0bf935c616 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -8,9 +8,25 @@ ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 +# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) +ARG PGBOUNCER_VERSION=1.25.2 +ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332 FROM $UV_IMAGE AS uvbin +FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder +ARG PGBOUNCER_VERSION +ARG PGBOUNCER_SHA256 +USER root +RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl +WORKDIR /build +RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \ + echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \ + tar xzf pgbouncer.tar.gz --strip-components=1 && \ + ./configure --prefix=/usr/local --with-openssl=/usr && \ + make -j"$(nproc)" pgbouncer && \ + install -m 0755 pgbouncer /usr/local/bin/pgbouncer + # Admin UI builder. Pinned to the build platform so the architecture-independent # Next.js static export compiles once natively even in a multi-arch build, # instead of once per target arch under QEMU. @@ -101,7 +117,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root # node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile +RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent +COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer WORKDIR /app ENV PATH="/app/.venv/bin:${PATH}" \ diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index edf20e8bbff..5d729046678 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -7,9 +7,25 @@ ARG PROXY_EXTRAS_SOURCE=published ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 +# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) +ARG PGBOUNCER_VERSION=1.25.2 +ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332 FROM $UV_IMAGE AS uvbin +FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder +ARG PGBOUNCER_VERSION +ARG PGBOUNCER_SHA256 +USER root +RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl +WORKDIR /build +RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \ + echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \ + tar xzf pgbouncer.tar.gz --strip-components=1 && \ + ./configure --prefix=/usr/local --with-openssl=/usr && \ + make -j"$(nproc)" pgbouncer && \ + install -m 0755 pgbouncer /usr/local/bin/pgbouncer + # Admin UI builder. Pinned to the build platform so the architecture-independent # Next.js static export compiles once natively even in a multi-arch build, # instead of once per target arch under QEMU. @@ -128,8 +144,9 @@ RUN for i in 1 2 3; do \ apk upgrade --no-cache && break || sleep 5; \ done && \ for i in 1 2 3; do \ - apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs && break || sleep 5; \ + apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs libevent && break || sleep 5; \ done +COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer # Copy only what runtime needs. The application is installed inside the venv; # the rest of the builder's /app is source and build metadata that must not diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 308d70a6b26..33d3791dbba 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,9 +1,25 @@ ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a +# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) +ARG PGBOUNCER_VERSION=1.25.2 +ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332 FROM $UV_IMAGE AS uvbin +FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder +ARG PGBOUNCER_VERSION +ARG PGBOUNCER_SHA256 +USER root +RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl +WORKDIR /build +RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \ + echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \ + tar xzf pgbouncer.tar.gz --strip-components=1 && \ + ./configure --prefix=/usr/local --with-openssl=/usr && \ + make -j"$(nproc)" pgbouncer && \ + install -m 0755 pgbouncer /usr/local/bin/pgbouncer + # ---------- Builder ---------- FROM $LITELLM_BUILD_IMAGE AS builder @@ -61,6 +77,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra bedrock-realtime \ --python python3.13 +# PYTHONPATH=/app makes the source tree shadow the installed package, so the +# compiled Rust extension must live next to the source or it is never imported. +RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/ + RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ prisma generate --schema=./schema.prisma @@ -73,7 +93,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \ + apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic libevent && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done @@ -90,15 +110,17 @@ ENV HOME=/home/nonroot \ COPY --from=builder --chown=nonroot:nonroot /app /app COPY --from=builder /opt/prisma /opt/prisma +COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ find /app/.venv -type d -path "*/tornado/test" -delete && \ chmod -R a+rX /opt/prisma && \ - python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" + python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \ + python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()" USER nonroot EXPOSE 4000/tcp -ENTRYPOINT ["sh", "-c", "exec /app/docker/component_entrypoint.sh uvicorn gateway.main:app --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"] +ENTRYPOINT ["sh", "-c", "exec /app/docker/component_entrypoint.sh python -m gateway.launch --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"] CMD ["--host", "0.0.0.0", "--port", "4000"] diff --git a/gateway/launch.py b/gateway/launch.py new file mode 100644 index 00000000000..6b28fafdcf6 --- /dev/null +++ b/gateway/launch.py @@ -0,0 +1,71 @@ +"""Gateway supervisor: assemble DATABASE_URL, start the in-container PgBouncer, then run uvicorn. + +``gateway/main.py`` assembles ``DATABASE_URL`` inside every uvicorn worker, which +is fine for a plain Postgres URL but not for the pooler: PgBouncer must be +started exactly once per pod, before the workers fork, and the workers must be +handed the loopback URL it listens on. A pre-existing ``DATABASE_URL`` wins in +``DatabaseURLSettings.apply_to_env`` under password auth, so setting it here is +enough for every worker to pick the pooled URL up unchanged. + +Run with: + python -m gateway.launch --workers 4 --host 0.0.0.0 --port 4000 +""" + +import os +import sys +from collections.abc import Callable, Mapping, Sequence +from typing import Final + +from uvicorn.main import main as uvicorn_main + +from litellm.proxy.db.db_url_settings import DatabaseURLSettings +from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, start_in_container_pgbouncer + +GATEWAY_APP: Final = "gateway.main:app" +KEEPALIVE_FLAG: Final = "--timeout-keep-alive" + + +def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str, ...]: + """Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly.""" + keepalive: Final = environ.get("KEEPALIVE_TIMEOUT") + if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv): + return (GATEWAY_APP, *argv) + return (GATEWAY_APP, *argv, KEEPALIVE_FLAG, keepalive) + + +def pool_database_url( + settings: DatabaseURLSettings, + pgbouncer: PgBouncerSettings, + environ: Mapping[str, str], +) -> str | PgBouncerError | None: + """Start the in-container PgBouncer and return its loopback URL, or None when ``pgbouncer.enabled`` is off. + + The upstream URL is whatever ``apply_to_env`` assembled from the discrete + ``DATABASE_*`` vars (or an operator-pinned ``DATABASE_URL``). Token auth is + rejected by the pooler itself, since it holds one password for its lifetime. + """ + if not pgbouncer.enabled: + return None + upstream_url: Final = environ.get("DATABASE_URL") + if upstream_url is None: + return PgBouncerError("LITELLM_PGBOUNCER_ENABLED is set but no DATABASE_URL could be assembled") + return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth_enabled=settings.token_auth() is not None) + + +def _serve(argv: Sequence[str]) -> None: + uvicorn_main(tuple(argv), prog_name="uvicorn") + + +def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: + settings: Final = DatabaseURLSettings.from_env() + settings.apply_to_env() + pooled_url: Final = pool_database_url(settings, PgBouncerSettings(), os.environ) + if isinstance(pooled_url, PgBouncerError): + sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}") + if pooled_url is not None: + os.environ["DATABASE_URL"] = pooled_url + serve(uvicorn_argv(argv, os.environ)) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/helm/litellm-helm/templates/deployment.yaml b/helm/litellm-helm/templates/deployment.yaml index f7c918a6827..834071eb9b2 100644 --- a/helm/litellm-helm/templates/deployment.yaml +++ b/helm/litellm-helm/templates/deployment.yaml @@ -117,6 +117,14 @@ spec: - name: DATABASE_URL_READ_REPLICA value: {{ .Values.db.readReplicaUrl | quote }} {{- end }} + {{- if .Values.db.connectionPool.enabled }} + - name: LITELLM_PGBOUNCER_ENABLED + value: "true" + - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: {{ .Values.db.connectionPool.maxDbConnections | quote }} + - name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + value: {{ .Values.db.connectionPool.maxClientConn | quote }} + {{- end }} - name: PROXY_MASTER_KEY valueFrom: secretKeyRef: diff --git a/helm/litellm-helm/tests/connection_pool_tests.yaml b/helm/litellm-helm/tests/connection_pool_tests.yaml new file mode 100644 index 00000000000..af23512dafc --- /dev/null +++ b/helm/litellm-helm/tests/connection_pool_tests.yaml @@ -0,0 +1,61 @@ +suite: test in-container connection pool +templates: + - deployment.yaml + - configmap-litellm.yaml +tests: + - it: should not emit pgbouncer env vars by default + template: deployment.yaml + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + value: "true" + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: "20" + + - it: should enable the pool with the default sizing when connectionPool.enabled is set + template: deployment.yaml + set: + db.connectionPool.enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + value: "true" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: "20" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + value: "1000" + + - it: should pass custom sizing through as strings next to the worker count + template: deployment.yaml + set: + numWorkers: 4 + db.connectionPool.enabled: true + db.connectionPool.maxDbConnections: 8 + db.connectionPool.maxClientConn: 400 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: "8" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + value: "400" + - contains: + path: spec.template.spec.containers[0].args + content: "4" diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 907f85de8fc..db596e2d68e 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -355,6 +355,20 @@ db: # only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime). readReplicaUrl: "" + # In-container connection pool (PgBouncer, transaction mode) shared by every + # worker in the pod. Without it each --num_workers worker opens its own + # connection_limit connections to Postgres, so a pod's footprint against the + # database's connection ceiling is workers x connection_limit and grows with + # every replica. With it, the pod holds at most maxDbConnections upstream + # connections no matter how many workers run; the workers connect to the pool + # over loopback, with no extra network hop. Migrations still go straight to + # Postgres. Starting profile for numWorkers: 4 is maxDbConnections: 20, so + # a database with a 5000-connection ceiling fits roughly 200 replicas. + connectionPool: + enabled: false + maxDbConnections: 20 + maxClientConn: 1000 + # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster. # The Stackgres Operator must already be installed within the target # Kubernetes cluster. diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index c459512c7b9..4ad3cfe0484 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -360,6 +360,23 @@ harmless no-op for the Job and authoritative for the app pods. {{- end }} {{- end -}} +{{/* +In-container PgBouncer env for the gateway container. Fails at render time under IAM or Entra auth: the pooler holds one static password for the life of the pod. +*/}} +{{- define "litellm.connectionPoolEnv" -}} +{{- if or .Values.database.writer.useIAMAuth .Values.database.writer.useAzureEntraAuth }} +{{- fail "database.connectionPool.enabled cannot be combined with database.writer.useIAMAuth or database.writer.useAzureEntraAuth: the in-container pgbouncer holds a static database password and cannot follow a rotating token. Disable the pool or use a static database password" }} +{{- end }} +{{- with .Values.database.connectionPool -}} +- name: LITELLM_PGBOUNCER_ENABLED + value: "true" +- name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: {{ required "database.connectionPool.maxDbConnections is required when the pool is enabled" .maxDbConnections | quote }} +- name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + value: {{ required "database.connectionPool.maxClientConn is required when the pool is enabled" .maxClientConn | quote }} +{{- end }} +{{- end -}} + {{/* PodDisruptionBudget shared by gateway, backend, and ui. diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index 9cb6b07e77b..1e9041f0a33 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -61,6 +61,9 @@ spec: - name: NUM_WORKERS value: {{ .Values.gateway.numWorkers | quote }} {{- end }} + {{- if .Values.database.connectionPool.enabled }} + {{- include "litellm.connectionPoolEnv" $ | nindent 12 }} + {{- end }} {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsEnv" . | nindent 12 }} {{- end }} diff --git a/helm/litellm/tests/connection_pool_tests.yaml b/helm/litellm/tests/connection_pool_tests.yaml new file mode 100644 index 00000000000..31be7575f3c --- /dev/null +++ b/helm/litellm/tests/connection_pool_tests.yaml @@ -0,0 +1,117 @@ +suite: test in-container connection pool env vars +templates: + - gateway/deployment.yaml + - gateway/configmap.yaml + - backend/deployment.yaml + - backend/configmap.yaml +values: + - ./values/required.yaml +tests: + - it: renders no pool env by default + template: gateway/deployment.yaml + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + any: true + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + any: true + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + any: true + + - it: enabled pool renders the three pgbouncer vars with the configured sizes + template: gateway/deployment.yaml + set: + gateway.numWorkers: 4 + database.connectionPool.enabled: true + database.connectionPool.maxDbConnections: 8 + database.connectionPool.maxClientConn: 250 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + value: "true" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: "8" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + value: "250" + - contains: + path: spec.template.spec.containers[0].env + content: + name: NUM_WORKERS + value: "4" + + - it: enabled pool uses the chart default sizes + template: gateway/deployment.yaml + set: + database.connectionPool.enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: "20" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + value: "1000" + + - it: backend never gets the pool env + template: backend/deployment.yaml + set: + database.connectionPool.enabled: true + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + any: true + + - it: pool with IAM auth fails at render time + template: gateway/deployment.yaml + set: + database.connectionPool.enabled: true + database.writer.useIAMAuth: true + asserts: + - failedTemplate: + errorMessage: "database.connectionPool.enabled cannot be combined with database.writer.useIAMAuth or database.writer.useAzureEntraAuth: the in-container pgbouncer holds a static database password and cannot follow a rotating token. Disable the pool or use a static database password" + + - it: pool with Entra auth fails at render time + template: gateway/deployment.yaml + set: + database.connectionPool.enabled: true + database.writer.useAzureEntraAuth: true + asserts: + - failedTemplate: + errorMessage: "database.connectionPool.enabled cannot be combined with database.writer.useIAMAuth or database.writer.useAzureEntraAuth: the in-container pgbouncer holds a static database password and cannot follow a rotating token. Disable the pool or use a static database password" + + - it: IAM auth without the pool still renders + template: gateway/deployment.yaml + set: + database.writer.useIAMAuth: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: IAM_TOKEN_DB_AUTH + value: "true" + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + any: true diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index be1e6d43987..45ab5229f38 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -225,6 +225,26 @@ database: usernameKey: username passwordKey: password + # In-container connection pool (PgBouncer, transaction mode) shared by every + # gateway worker in the pod. Without it each of the `gateway.numWorkers` + # workers opens its own Prisma pool straight to Postgres, so a pod's + # footprint against the database's connection ceiling is + # numWorkers x connection_limit and grows with every replica. With it, the + # pod holds at most maxDbConnections upstream connections no matter how many + # workers run; the workers connect to the pool over loopback, with no extra + # network hop. The chart emits LITELLM_PGBOUNCER_ENABLED / + # LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS / LITELLM_PGBOUNCER_MAX_CLIENT_CONN on + # the gateway container only: the backend runs a single worker and the + # migrations Job must keep a direct connection. The pool holds a static + # password, so it cannot be combined with `database.writer.useIAMAuth` or + # `useAzureEntraAuth` (rendering fails). Starting profile for + # `gateway.numWorkers: 4` is maxDbConnections: 20, so a database with a + # 5000-connection ceiling fits roughly 200 gateway replicas. + connectionPool: + enabled: false + maxDbConnections: 20 + maxClientConn: 1000 + # Optional Redis. Leave host empty to disable. # # This is the proxy's coordination store: cross-pod tpm/rpm rate limits, spend diff --git a/litellm/proxy/db/pgbouncer.py b/litellm/proxy/db/pgbouncer.py new file mode 100644 index 00000000000..7792867600c --- /dev/null +++ b/litellm/proxy/db/pgbouncer.py @@ -0,0 +1,525 @@ +"""In-container PgBouncer shared by every proxy worker. + +Each uvicorn worker owns a Prisma query engine with its own pool of +``connection_limit`` server connections, so the connections a pod holds open +against Postgres scale as ``workers * connection_limit`` and a database with a +fixed connection ceiling runs out of room as pods and workers are added. + +When ``LITELLM_PGBOUNCER_ENABLED`` is set, the supervisor process starts one +PgBouncer next to the workers (no extra network hop: it listens on loopback +inside the pod) in transaction pooling mode, points ``DATABASE_URL`` at it +with ``pgbouncer=true`` so Prisma stops using server-side prepared statements, +and keeps it running for the life of the proxy. Every worker's pool then +becomes cheap client connections to PgBouncer while the upstream connection +count is capped at ``LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS`` per pod, no matter +how many workers run. + +Migrations and the schema diff run in the supervisor before the pooler is +started, so they always go straight to Postgres. ``DATABASE_URL_READ_REPLICA`` +is left untouched. + +The pooler holds the database password from startup, so it cannot be combined +with ``IAM_TOKEN_DB_AUTH`` or ``AZURE_POSTGRESQL_AUTH``: those rotate the +password inside every worker on their own schedule, and PgBouncer would keep +authenticating upstream with the expired token. +""" + +from __future__ import annotations + +import atexit +import os +import re +import shlex +import shutil +import socket +import subprocess +import tempfile +import threading +import time +import urllib.parse +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + +from litellm._logging import verbose_proxy_logger +from litellm.proxy.db.token_auth import AZURE_POSTGRESQL_AUTH_ENV_VAR, IAM_TOKEN_DB_AUTH_ENV_VAR + +PGBOUNCER_ENV_PREFIX: Final = "LITELLM_PGBOUNCER_" +PGBOUNCER_LISTEN_ADDR: Final = "127.0.0.1" +PGBOUNCER_INI_NAME: Final = "pgbouncer.ini" +PGBOUNCER_USERLIST_NAME: Final = "userlist.txt" +PGBOUNCER_CA_NAME: Final = "server-ca.pem" +PGBOUNCER_RESTART_DELAY_SECONDS: Final = 1.0 +PGBOUNCER_READY_TIMEOUT_SECONDS: Final = 15.0 +PGBOUNCER_STOP_GRACE_SECONDS: Final = 10.0 +PGBOUNCER_UNPRIVILEGED_USER: Final = "nobody" +PGBOUNCER_MIN_VERSION: Final = (1, 19) +PGBOUNCER_VERSION_PATTERN: Final = re.compile(r"PgBouncer (\d+)\.(\d+)") +PGBOUNCER_LIST_DELIMITER_PATTERN: Final = re.compile(r"[,\s]") +PGBOUNCER_TOKEN_AUTH_CONFLICT: Final = ( + f"the in-container pgbouncer cannot be combined with {IAM_TOKEN_DB_AUTH_ENV_VAR} or " + f"{AZURE_POSTGRESQL_AUTH_ENV_VAR}: each worker rotates the database password on its own schedule and the pooler " + "would keep using the expired token upstream. Disable the pooler or use a static database password" +) + +# Prisma's client-side TLS params describe the hop to Postgres, which becomes +# PgBouncer's server side. They move into ``server_tls_*`` and must not stay on +# the loopback URL: the listener speaks plain TCP and Prisma would refuse it +# under ``sslmode=require`` or ``channel_binding=require``. +PRISMA_TLS_PARAM_KEYS: Final[frozenset[str]] = frozenset( + {"sslmode", "sslcert", "sslaccept", "sslidentity", "sslpassword", "channel_binding", "gssencmode"} +) +POOLED_URL_DROPPED_KEYS: Final[frozenset[str]] = PRISMA_TLS_PARAM_KEYS | frozenset(("options", "pgbouncer")) +PGBOUNCER_SSLMODES: Final[frozenset[str]] = frozenset( + {"disable", "allow", "prefer", "require", "verify-ca", "verify-full"} +) + + +class PgBouncerSettings(BaseSettings): + """``LITELLM_PGBOUNCER_*`` env vars, read once in the supervisor.""" + + model_config = SettingsConfigDict( + env_prefix=PGBOUNCER_ENV_PREFIX, case_sensitive=False, extra="ignore", frozen=True + ) + + enabled: bool = False + port: int = Field(default=6432, ge=1, le=65535) + max_db_connections: int = Field(default=20, ge=1) + max_client_conn: int = Field(default=1000, ge=1) + binary: str = "pgbouncer" + + +@dataclass(frozen=True, slots=True) +class PgBouncerPlan: + ini: str + userlist: str + pooled_url: str + ca_source: str | None = None + + +@dataclass(frozen=True, slots=True) +class PgBouncerError: + reason: str + + +def _single_quoted(value: str) -> str: + """Quote for SQL and for PgBouncer's ``[databases]`` connection string: both double a literal ``'``.""" + return "'" + value.replace("'", "''") + "'" + + +def _userlist_quote(value: str) -> str: + return '"' + value.replace('"', '""') + '"' + + +def _option_settings(tokens: Sequence[str]) -> tuple[str, ...] | None: + """The ``name=value`` settings in a libpq ``options`` string, or None if it holds anything else. + + Accepts ``-c name=value``, ``-cname=value`` and ``--name=value``; a + detached ``-c`` is folded into the token that follows it first. + """ + folded: Final = tuple( + f"-c{tokens[index + 1]}" if token == "-c" and index + 1 < len(tokens) else token + for index, token in enumerate(tokens) + if index == 0 or tokens[index - 1] != "-c" + ) + settings: Final = tuple(token[2:] for token in folded if token.startswith(("-c", "--")) and "=" in token[2:]) + return settings if len(settings) == len(folded) else None + + +def _connect_query(options: str) -> str | PgBouncerError: + """Turn Prisma's ``options=-c name=value ...`` startup param into ``SET`` statements. + + PgBouncer rejects any ``-c`` setting in ``options`` that is not one of the + handful it tracks (``statement_timeout`` and ``lock_timeout`` are not), so + the settings are applied to each new server connection instead. Every + client shares them, which is what the single ``DATABASE_URL`` gave anyway. + """ + settings: Final = _option_settings(tuple(shlex.split(options))) + if settings is None: + return PgBouncerError(f"cannot translate the DATABASE_URL options {options!r} into PgBouncer settings") + return "; ".join( + f"SET {name.strip()} TO {_single_quoted(value.strip())}" + for name, value in (setting.split("=", 1) for setting in settings) + ) + + +def _server_tls_settings(sslmode: str, sslcert: str, sslaccept: str, ca_path: Path) -> tuple[str, ...] | PgBouncerError: + """``server_tls_*`` lines naming ``ca_path``, the runtime-dir copy of the bundle: the original (or the + 0600 root pinned by ``pin_bundle_root``) is often unreadable for the user PgBouncer drops to.""" + if sslmode not in PGBOUNCER_SSLMODES: + return PgBouncerError(f"unsupported sslmode {sslmode!r} on DATABASE_URL") + verify: Final = sslmode in ("verify-ca", "verify-full") or (sslmode == "require" and sslaccept == "strict") + if verify and not sslcert: + return PgBouncerError( + "DATABASE_URL asks for a verified TLS connection but names no CA bundle; " + "add sslcert= (or sslrootcert=) so the in-container PgBouncer can verify Postgres" + ) + mode: Final = "verify-full" if verify else sslmode + return (f"server_tls_sslmode = {mode}", *((f"server_tls_ca_file = {ca_path}",) if sslcert else ())) + + +def plan_pgbouncer( + upstream_url: str, + settings: PgBouncerSettings, + runtime_dir: Path, + run_as_user: str | None, +) -> PgBouncerPlan | PgBouncerError: + """Render the PgBouncer config for ``upstream_url`` and the loopback URL Prisma uses instead. + + Params describing Prisma's own pool (``connection_limit``, ``pool_timeout``, + ...) stay on the pooled URL; the TLS params and ``options`` describe the hop + to Postgres and move into the PgBouncer config. ``run_as_user`` is the + unprivileged user PgBouncer drops to when the proxy runs as root, which + PgBouncer itself refuses to do. + """ + parsed: Final = urllib.parse.urlsplit(upstream_url) + params: Final[Mapping[str, str]] = MappingProxyType( + dict(urllib.parse.parse_qsl(parsed.query, keep_blank_values=True)) + ) + dbname: Final = urllib.parse.unquote(parsed.path.lstrip("/")) + username: Final = urllib.parse.unquote(parsed.username or "") + password: Final = None if parsed.password is None else urllib.parse.unquote(parsed.password) + if not parsed.hostname or not username or password is None or not dbname: + return PgBouncerError( + "DATABASE_URL must carry a host, user, password and database name for the in-container PgBouncer" + ) + if PGBOUNCER_LIST_DELIMITER_PATTERN.search(username): + return PgBouncerError( + f"the database user {username!r} cannot be named in PgBouncer's stats_users list: " + "PgBouncer splits list settings on commas and whitespace and has no quoting for them" + ) + if "sslidentity" in params: + return PgBouncerError("client certificates (sslidentity) are not supported with the in-container PgBouncer") + tls: Final = _server_tls_settings( + params.get("sslmode", "prefer"), + params.get("sslcert", ""), + params.get("sslaccept", ""), + runtime_dir / PGBOUNCER_CA_NAME, + ) + if isinstance(tls, PgBouncerError): + return tls + connect_query: Final = _connect_query(params["options"]) if params.get("options") else "" + if isinstance(connect_query, PgBouncerError): + return connect_query + upstream: Final = " ".join( + ( + f"host={_single_quoted(parsed.hostname)}", + f"port={parsed.port or 5432}", + f"dbname={_single_quoted(dbname)}", + f"user={_single_quoted(username)}", + f"password={_single_quoted(password)}", + *((f"connect_query={_single_quoted(connect_query)}",) if connect_query else ()), + ) + ) + ini: Final = "\n".join( + ( + "[databases]", + f"{dbname} = {upstream}", + "", + "[pgbouncer]", + f"listen_addr = {PGBOUNCER_LISTEN_ADDR}", + f"listen_port = {settings.port}", + f"unix_socket_dir = {runtime_dir}", + f"auth_file = {runtime_dir / PGBOUNCER_USERLIST_NAME}", + "auth_type = scram-sha-256", + f"stats_users = {username}", + "pool_mode = transaction", + f"max_client_conn = {settings.max_client_conn}", + f"default_pool_size = {settings.max_db_connections}", + f"max_db_connections = {settings.max_db_connections}", + "ignore_startup_parameters = extra_float_digits", + *tls, + *((f"user = {run_as_user}",) if run_as_user else ()), + "", + ) + ) + userlist: Final = f"{_userlist_quote(username)} {_userlist_quote(password)}\n" + pooled_query: Final = urllib.parse.urlencode( + (*((key, value) for key, value in params.items() if key not in POOLED_URL_DROPPED_KEYS), ("pgbouncer", "true")) + ) + credentials: Final = f"{urllib.parse.quote(username, safe='')}:{urllib.parse.quote(password, safe='')}" + pooled_url: Final = urllib.parse.urlunsplit( + parsed._replace(netloc=f"{credentials}@{PGBOUNCER_LISTEN_ADDR}:{settings.port}", query=pooled_query) + ) + return PgBouncerPlan(ini=ini, userlist=userlist, pooled_url=pooled_url, ca_source=params.get("sslcert") or None) + + +def write_pgbouncer_files(plan: PgBouncerPlan, runtime_dir: Path, run_as_user: str | None) -> Path | PgBouncerError: + """Write the ini, userlist (both hold the password, so mode 0600) and CA copy, and return the ini path. + + ``run_as_user`` is the user PgBouncer drops to when started as root; it has + to own the files it re-reads on reload and the socket directory. + """ + ini_path: Final = runtime_dir / PGBOUNCER_INI_NAME + userlist_path: Final = runtime_dir / PGBOUNCER_USERLIST_NAME + ca_path: Final = runtime_dir / PGBOUNCER_CA_NAME + if plan.ca_source is not None: + try: + shutil.copyfile(plan.ca_source, ca_path) + except OSError as error: + return PgBouncerError(f"cannot read the CA bundle {plan.ca_source!r} named by sslcert: {error}") + for path, content in ((userlist_path, plan.userlist), (ini_path, plan.ini)): + path.touch(mode=0o600) + path.write_text(content, encoding="utf-8") + if run_as_user is not None: + runtime_dir.chmod(0o700) + for path in (runtime_dir, ini_path, userlist_path, *((ca_path,) if plan.ca_source is not None else ())): + shutil.chown(path, user=run_as_user) + return ini_path + + +def _port_open(port: int) -> bool: + try: + with socket.create_connection((PGBOUNCER_LISTEN_ADDR, port), timeout=0.5): + return True + except OSError: + return False + + +def _unix_socket_open(path: Path) -> bool: + with socket.socket(socket.AF_UNIX) as probe: + probe.settimeout(0.5) + try: + probe.connect(str(path)) + except OSError: + return False + return True + + +def unix_socket_path(runtime_dir: Path, port: int) -> Path: + return runtime_dir / f".s.PGSQL.{port}" + + +def pgbouncer_version(binary: str) -> tuple[int, int] | PgBouncerError: + """``(major, minor)`` from `` --version``. + + Readiness relies on PgBouncer exiting when it cannot bind its TCP port, + which it does from 1.19 on. Older releases log a warning and serve the unix + socket alone, so their socket would vouch for a port held by someone else. + """ + try: + output: Final = subprocess.run( + (binary, "--version"), capture_output=True, text=True, check=False, timeout=10 + ).stdout + except (OSError, subprocess.TimeoutExpired) as run_error: + return PgBouncerError(f"could not run {binary!r} --version: {run_error}") + found: Final = PGBOUNCER_VERSION_PATTERN.search(output) + if found is None: + return PgBouncerError(f"{binary!r} --version did not report a PgBouncer version: {output.strip()!r}") + return int(found[1]), int(found[2]) + + +def _end(process: subprocess.Popen[bytes]) -> None: + if process.poll() is not None: + return + process.terminate() + try: + process.wait(timeout=PGBOUNCER_STOP_GRACE_SECONDS) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +class PgBouncerProcess: + """Runs ``argv`` as a foreground child and restarts it whenever it exits on its own. + + Prisma reconnects by itself after a failed query, so a PgBouncer crash + costs the requests in flight plus one failed query per idle pooled + connection the crash severed, and nothing else once the replacement is + listening again. A replacement that cannot be spawned, finds its port + taken, exits again or never starts listening is retried every + ``restart_delay_seconds`` until ``stop`` is called. + + A connect probe of ``port`` cannot tell the child from another process + that grabbed the port after the availability check, so readiness also + needs ``socket_path``: the unix socket PgBouncer creates in the private + runtime directory, which it only does once every TCP listener is bound + (PgBouncer 1.19 or newer, see ``pgbouncer_version``). + """ + + def __init__( + self, + argv: Sequence[str], + port: int, + socket_path: Path, + restart_delay_seconds: float = PGBOUNCER_RESTART_DELAY_SECONDS, + ready_timeout_seconds: float = PGBOUNCER_READY_TIMEOUT_SECONDS, + ) -> None: + self.argv: Final = tuple(argv) + self.port: Final = port + self.socket_path: Final = socket_path + self.restart_delay_seconds: Final = restart_delay_seconds + self.ready_timeout_seconds: Final = ready_timeout_seconds + self._stopping: Final = threading.Event() + self._lock: Final = threading.Lock() + self._process: subprocess.Popen[bytes] | None = None + + @property + def pid(self) -> int | None: + with self._lock: + return None if self._process is None else self._process.pid + + def _spawn(self) -> subprocess.Popen[bytes] | PgBouncerError | None: + """Start a child, or None once ``stop`` ran; both take the lock so no child can slip in after a stop. + + The port has to be free first: a listener that is already there would + pass the readiness check while the child fails to bind. + """ + with self._lock: + if self._stopping.is_set(): + return None + if _port_open(self.port): + return PgBouncerError(f"{PGBOUNCER_LISTEN_ADDR}:{self.port} is already in use by another process") + try: + process: Final = subprocess.Popen(self.argv) + except OSError as spawn_error: + return PgBouncerError(f"could not start {self.argv[0]!r}: {spawn_error}") + self._process = process + return process + + def _wait_ready(self, process: subprocess.Popen[bytes]) -> PgBouncerError | None: + deadline: Final = time.monotonic() + self.ready_timeout_seconds + while time.monotonic() < deadline: + if process.poll() is not None: + return PgBouncerError(f"pgbouncer exited with status {process.returncode} during startup") + if _port_open(self.port) and _unix_socket_open(self.socket_path): + return None + time.sleep(0.1) + if _port_open(self.port): + return PgBouncerError( + f"{PGBOUNCER_LISTEN_ADDR}:{self.port} is served by another process, not the pgbouncer that was started" + ) + return PgBouncerError( + f"pgbouncer did not start listening on {PGBOUNCER_LISTEN_ADDR}:{self.port} " + f"within {self.ready_timeout_seconds:.0f}s" + ) + + def start(self) -> PgBouncerError | None: + """Spawn PgBouncer, wait until it listens on port and unix socket, then supervise it from a daemon thread.""" + process: Final = self._spawn() + if process is None: + return PgBouncerError("pgbouncer was stopped before it started") + if isinstance(process, PgBouncerError): + return process + not_ready: Final = self._wait_ready(process) + if not_ready is not None: + self.stop() + return not_ready + self._watch(process) + return None + + def _watch(self, process: subprocess.Popen[bytes]) -> None: + threading.Thread( + target=self._supervise, args=(process,), daemon=True, name="litellm-pgbouncer-supervisor" + ).start() + + def _supervise(self, process: subprocess.Popen[bytes]) -> None: + status: Final = process.wait() + if self._stopping.is_set(): + return + verbose_proxy_logger.error( + "In-container pgbouncer (pid %s) exited with status %s; restarting in %.1fs.", + process.pid, + status, + self.restart_delay_seconds, + ) + self._restart_after_delay() + + def _restart_after_delay(self) -> None: + time.sleep(self.restart_delay_seconds) + process: Final = self._spawn() + if process is None: + return + if isinstance(process, PgBouncerError): + self._retry_restart(process.reason) + return + not_ready: Final = self._wait_ready(process) + if not_ready is None: + self._watch(process) + return + _end(process) + self._retry_restart(not_ready.reason) + + def _retry_restart(self, reason: str) -> None: + if self._stopping.is_set(): + return + verbose_proxy_logger.error( + "In-container pgbouncer could not be restarted (%s); retrying in %.1fs.", reason, self.restart_delay_seconds + ) + threading.Thread(target=self._restart_after_delay, daemon=True, name="litellm-pgbouncer-supervisor").start() + + def stop(self) -> None: + with self._lock: + self._stopping.set() + process: Final = self._process + if process is not None: + _end(process) + + +def _only_in_this_process(action: Callable[[], None]) -> Callable[[], None]: + """An exit hook that does nothing in a forked child, which inherits the parent's ``atexit`` table.""" + owner_pid: Final = os.getpid() + + def run() -> None: + if os.getpid() == owner_pid: + action() + + return run + + +def start_in_container_pgbouncer( + settings: PgBouncerSettings, + upstream_url: str, + token_auth_enabled: bool = False, + register_exit_hook: Callable[[Callable[[], None]], object] = atexit.register, +) -> str | PgBouncerError: + """Start the pooler for ``upstream_url`` and return the loopback URL the workers must use. + + The pooler lives as long as this process: it is stopped from the exit hooks + once the worker manager has returned, and only by the process that started + it (gunicorn forks its workers, so they carry the hooks too). PgBouncer + refuses to run as root, so a root proxy (the default image) has it drop to + ``nobody``. + """ + if token_auth_enabled: + return PgBouncerError(PGBOUNCER_TOKEN_AUTH_CONFLICT) + version: Final = pgbouncer_version(settings.binary) + if isinstance(version, PgBouncerError): + return version + if version < PGBOUNCER_MIN_VERSION: + return PgBouncerError( + f"PgBouncer {version[0]}.{version[1]} keeps running after failing to bind its TCP port, so the proxy " + f"cannot tell it apart from another listener; {PGBOUNCER_MIN_VERSION[0]}.{PGBOUNCER_MIN_VERSION[1]} " + "or newer is required" + ) + runtime_dir: Final = Path(tempfile.mkdtemp(prefix="litellm-pgbouncer-")) + register_exit_hook(_only_in_this_process(lambda: shutil.rmtree(runtime_dir, ignore_errors=True))) + run_as_user: Final = PGBOUNCER_UNPRIVILEGED_USER if os.geteuid() == 0 else None + plan: Final = plan_pgbouncer(upstream_url, settings, runtime_dir, run_as_user) + if isinstance(plan, PgBouncerError): + return plan + ini_path: Final = write_pgbouncer_files(plan, runtime_dir, run_as_user) + if isinstance(ini_path, PgBouncerError): + return ini_path + pooler: Final = PgBouncerProcess( + argv=(settings.binary, str(ini_path)), + port=settings.port, + socket_path=unix_socket_path(runtime_dir, settings.port), + ) + failed: Final = pooler.start() + if failed is not None: + return failed + register_exit_hook(_only_in_this_process(pooler.stop)) + verbose_proxy_logger.info( + "In-container pgbouncer (pid %s) listening on %s:%s; capping this pod at %s upstream database connections.", + pooler.pid, + PGBOUNCER_LISTEN_ADDR, + settings.port, + settings.max_db_connections, + ) + return plan.pooled_url diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index e245367b1b4..16d76ff0415 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -18,6 +18,7 @@ from pydantic import BaseModel, ConfigDict import litellm from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY +from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, start_in_container_pgbouncer from litellm.proxy.db.query_engine_reaper import start_query_engine_reaper if TYPE_CHECKING: @@ -1377,6 +1378,21 @@ def run_server( print( f"Unable to connect to DB. DATABASE_URL found in environment, but prisma package not found." # noqa: F541 ) + pgbouncer_settings: Final = PgBouncerSettings() + upstream_database_url: Final = os.getenv("DATABASE_URL") + if pgbouncer_settings.enabled and upstream_database_url is not None: + pooled_database_url: Final = start_in_container_pgbouncer( + pgbouncer_settings, upstream_database_url, token_auth_enabled=wants_rds_iam or wants_azure_entra + ) + if isinstance(pooled_database_url, PgBouncerError): + print( + f"\033[1;31mLiteLLM Proxy: LITELLM_PGBOUNCER_ENABLED is set but the in-container pgbouncer " + f"could not start: {pooled_database_url.reason}\033[0m", + file=sys.stderr, + flush=True, + ) + sys.exit(1) + os.environ["DATABASE_URL"] = pooled_database_url if port == 4000 and ProxyInitializationHelpers._is_port_in_use(port): port = random.randint(1024, 49152) if prometheus_metrics_port == port: diff --git a/terraform/litellm/aws/README.md b/terraform/litellm/aws/README.md index 6d3e0269a15..986428151e0 100644 --- a/terraform/litellm/aws/README.md +++ b/terraform/litellm/aws/README.md @@ -258,6 +258,39 @@ gateway_metrics_port = 4001 gateway_metrics_scrape_cidrs = ["10.0.0.0/16"] ``` +### In-container connection pool + +Each of the `gateway_num_workers` uvicorn workers opens its own Prisma pool +straight to Postgres, so one task holds `workers x connection_limit` +connections and the fleet's footprint against the database ceiling grows with +every task. `gateway_connection_pool_enabled` runs a PgBouncer (transaction +mode, loopback) inside the gateway container that all workers share, capping +the task at `gateway_pool_max_db_connections` upstream connections however +many workers it runs. `gateway_pool_max_client_conn` bounds the worker-side +connections the pooler accepts. The module sets +`LITELLM_PGBOUNCER_ENABLED`, `LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS` and +`LITELLM_PGBOUNCER_MAX_CLIENT_CONN` on the gateway container only; the backend +and the migration task keep the direct connection. + +```hcl +create_database = false +database_url = "postgresql://litellm:@db.internal:5432/litellm" +gateway_num_workers = 4 +gateway_connection_pool_enabled = true +gateway_pool_max_db_connections = 20 +gateway_pool_max_client_conn = 1000 +``` + +The pool needs a static database password, so it is only valid with an +existing database via `database_url`. The module-created Aurora authenticates +with rotating IAM tokens (see [Aurora + IAM auth](#aurora--iam-auth)), which +the pooler cannot follow, and `terraform plan` rejects that combination. + +The componentized `gateway_image` starts through `python -m gateway.launch`, +which reads these variables, starts the pooler once per task and hands the +workers its loopback URL; the classic `litellm` image honours them the same +way. + ### Scaling the gateway on requests and tokens By default the gateway service target-tracks CPU (`gateway_cpu_target`) and diff --git a/terraform/litellm/aws/ecs.tf b/terraform/litellm/aws/ecs.tf index aa2c3d558e2..149842c14ff 100644 --- a/terraform/litellm/aws/ecs.tf +++ b/terraform/litellm/aws/ecs.tf @@ -213,6 +213,12 @@ locals { # otherwise we keep the image's ENTRYPOINT and only override `command`. gateway_uvicorn_args = "--host 0.0.0.0 --port 4000 --workers ${var.gateway_num_workers}" + gateway_pool_env = var.gateway_connection_pool_enabled ? [ + { name = "LITELLM_PGBOUNCER_ENABLED", value = "true" }, + { name = "LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS", value = tostring(var.gateway_pool_max_db_connections) }, + { name = "LITELLM_PGBOUNCER_MAX_CLIENT_CONN", value = tostring(var.gateway_pool_max_client_conn) }, + ] : [] + metrics_enabled = var.gateway_metrics_port != null metrics_multiproc_dir = "/tmp/litellm_prometheus_multiproc" metrics_volume = "prometheus-multiproc" @@ -253,7 +259,7 @@ locals { backend_uvicorn_args = "--host 0.0.0.0 --port 4001" - gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn gateway.main:app ${local.gateway_uvicorn_args};; *) exec uvicorn gateway.main:app ${local.gateway_uvicorn_args};; esac" + gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run python -m gateway.launch ${local.gateway_uvicorn_args};; *) exec python -m gateway.launch ${local.gateway_uvicorn_args};; esac" backend_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn backend.main:app ${local.backend_uvicorn_args};; *) exec uvicorn backend.main:app ${local.backend_uvicorn_args};; esac" gateway_proxy_overrides = local.proxy_config_enabled ? { @@ -298,6 +304,11 @@ resource "aws_ecs_task_definition" "gateway" { ) error_message = "billing_metrics_client_cert_pem and billing_metrics_client_key_pem are both required when billing_metrics_endpoint is set." } + + precondition { + condition = !var.gateway_connection_pool_enabled || local.byo_database + error_message = "gateway_connection_pool_enabled requires an existing database via database_url with create_database = false: the module-created Aurora authenticates with IAM tokens, which the in-container pgbouncer cannot follow because it holds a static database password." + } } family = "${local.name}-gateway" @@ -323,6 +334,7 @@ resource "aws_ecs_task_definition" "gateway" { local.gateway_extra_env_list, local.proxy_config_env, local.metrics_env, + local.gateway_pool_env, ) secrets = concat(local.shared_secrets, local.gateway_extra_secrets_list) mountPoints = local.metrics_mount_points diff --git a/terraform/litellm/aws/tests/connection_pool.tftest.hcl b/terraform/litellm/aws/tests/connection_pool.tftest.hcl new file mode 100644 index 00000000000..38aa7051134 --- /dev/null +++ b/terraform/litellm/aws/tests/connection_pool.tftest.hcl @@ -0,0 +1,123 @@ +# Plan-only coverage for the in-container PgBouncer knobs on the gateway task. +# `mock_provider` keeps this offline: no AWS credentials, no API calls, no +# resources. Run from terraform/litellm/aws with `terraform test`. + +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } +} +mock_provider "random" {} + +variables { + region = "us-east-1" + tenant = "acme" + env = "test" + azs = ["us-east-1a", "us-east-1b"] + allow_plaintext_alb = true +} + +run "pool_off_by_default" { + command = plan + + assert { + condition = length(local.gateway_pool_env) == 0 + error_message = "The gateway must get no LITELLM_PGBOUNCER_* env unless gateway_connection_pool_enabled is set." + } +} + +run "pool_enabled_renders_the_three_vars_with_configured_sizes" { + command = plan + + variables { + create_database = false + database_url = "postgresql://litellm:pw@db.internal:5432/litellm" + gateway_num_workers = 4 + gateway_connection_pool_enabled = true + gateway_pool_max_db_connections = 8 + gateway_pool_max_client_conn = 250 + } + + assert { + condition = alltrue([ + length(local.gateway_pool_env) == 3, + local.gateway_pool_env[0].name == "LITELLM_PGBOUNCER_ENABLED" && local.gateway_pool_env[0].value == "true", + local.gateway_pool_env[1].name == "LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS" && local.gateway_pool_env[1].value == "8", + local.gateway_pool_env[2].name == "LITELLM_PGBOUNCER_MAX_CLIENT_CONN" && local.gateway_pool_env[2].value == "250", + ]) + error_message = "The pool env must carry the enabled flag and the configured sizes as strings." + } +} + +run "pool_enabled_uses_the_module_default_sizes" { + command = plan + + variables { + create_database = false + database_url = "postgresql://litellm:pw@db.internal:5432/litellm" + gateway_connection_pool_enabled = true + } + + assert { + condition = alltrue([ + length(local.gateway_pool_env) == 3, + local.gateway_pool_env[1].name == "LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS" && local.gateway_pool_env[1].value == "20", + local.gateway_pool_env[2].name == "LITELLM_PGBOUNCER_MAX_CLIENT_CONN" && local.gateway_pool_env[2].value == "1000", + ]) + error_message = "The pool env must fall back to the module defaults of 20 upstream and 1000 client connections." + } +} + +run "gateway_starts_through_the_pool_aware_launcher" { + command = plan + + variables { + gateway_num_workers = 4 + } + + assert { + condition = alltrue([ + strcontains(local.gateway_launch_cmd, "exec python -m gateway.launch --host 0.0.0.0 --port 4000 --workers 4"), + strcontains(local.gateway_launch_cmd, "exec ddtrace-run python -m gateway.launch --host 0.0.0.0 --port 4000 --workers 4"), + !strcontains(local.gateway_launch_cmd, "uvicorn gateway.main:app"), + local.gateway_proxy_overrides.command[0] == local.gateway_launch_cmd, + ]) + error_message = "The gateway must start through gateway.launch (with and without ddtrace) so the pooler starts once before uvicorn forks the workers." + } +} + +run "pool_with_module_created_iam_aurora_fails_at_plan" { + command = plan + + variables { + gateway_connection_pool_enabled = true + } + + expect_failures = [ + aws_ecs_task_definition.gateway, + ] +} + +run "pool_without_any_database_fails_at_plan" { + command = plan + + variables { + create_database = false + gateway_connection_pool_enabled = true + } + + expect_failures = [ + aws_ecs_task_definition.gateway, + ] +} + +run "module_created_iam_aurora_without_the_pool_still_plans" { + command = plan + + assert { + condition = contains(local.managed_db_env, { name = "IAM_TOKEN_DB_AUTH", value = "true" }) + error_message = "Without the pool the module-created Aurora must keep IAM token auth." + } +} diff --git a/terraform/litellm/aws/variables.tf b/terraform/litellm/aws/variables.tf index 10a9b392673..ec8363dbbaf 100644 --- a/terraform/litellm/aws/variables.tf +++ b/terraform/litellm/aws/variables.tf @@ -200,6 +200,45 @@ variable "gateway_num_workers" { } } +variable "gateway_connection_pool_enabled" { + description = <<-EOT + Run an in-container PgBouncer (transaction mode, loopback) in each gateway + task, shared by every uvicorn worker. Without it each of the + `gateway_num_workers` workers opens its own Prisma pool straight to + Postgres, so a task's footprint against the database connection ceiling is + workers x connection_limit and grows with every task. Sets + LITELLM_PGBOUNCER_ENABLED / LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS / + LITELLM_PGBOUNCER_MAX_CLIENT_CONN on the gateway container only. Requires + an existing database via `database_url`: the module-created Aurora + authenticates with IAM tokens, which the pooler cannot follow because it + holds one static password for the life of the task. + EOT + type = bool + default = false +} + +variable "gateway_pool_max_db_connections" { + description = "Upstream Postgres connections one gateway task may hold when gateway_connection_pool_enabled is set, regardless of gateway_num_workers. 20 suits 4 workers; a 5000-connection database then fits roughly 200 tasks." + type = number + default = 20 + + validation { + condition = var.gateway_pool_max_db_connections >= 1 + error_message = "gateway_pool_max_db_connections must be >= 1." + } +} + +variable "gateway_pool_max_client_conn" { + description = "Client connections the in-container PgBouncer accepts from the gateway workers when gateway_connection_pool_enabled is set." + type = number + default = 1000 + + validation { + condition = var.gateway_pool_max_client_conn >= 1 + error_message = "gateway_pool_max_client_conn must be >= 1." + } +} + variable "backend_cpu" { description = "Fargate CPU units for the backend task (1024 = 1 vCPU)." type = number diff --git a/terraform/litellm/gcp/README.md b/terraform/litellm/gcp/README.md index 66095ec0108..9c71d3e15b7 100644 --- a/terraform/litellm/gcp/README.md +++ b/terraform/litellm/gcp/README.md @@ -287,6 +287,40 @@ the gateway on GKE with the Helm chart's `targetTokensPerSecond` (see "Dependencies only" below) rather than wiring the counter into Cloud Monitoring, which the autoscaler would ignore +### In-container connection pool + +Each of the `gateway_num_workers` uvicorn workers opens its own Prisma pool +straight to Cloud SQL, so one instance holds `workers x connection_limit` +connections and the fleet's footprint against the database ceiling grows with +every instance Cloud Run adds. `gateway_connection_pool_enabled` runs a +PgBouncer (transaction mode, loopback) inside the gateway container that all +workers share, capping the instance at `gateway_pool_max_db_connections` +upstream connections however many workers it runs. +`gateway_pool_max_client_conn` bounds the worker-side connections the pooler +accepts. The module sets `LITELLM_PGBOUNCER_ENABLED`, +`LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS` and `LITELLM_PGBOUNCER_MAX_CLIENT_CONN` +on the gateway service only; the backend service and the migrations job keep +the direct connection + +```hcl +gateway_num_workers = 4 +gateway_connection_pool_enabled = true +gateway_pool_max_db_connections = 20 +gateway_pool_max_client_conn = 1000 +``` + +The pooler holds one static database password for the life of the instance. +This stack authenticates to Cloud SQL with the Secret Manager password (see +[Database authentication](#database-authentication)), so nothing else is +needed; a Cloud SQL Auth Proxy sidecar with IAM auth would not work with the +pool + +The gateway container starts through `python -m gateway.launch` (the +componentized image's own entrypoint) rather than `uvicorn` directly. The +launcher reads these variables, starts the pooler once per instance before +uvicorn forks the workers and hands them its loopback `DATABASE_URL`. It also +honours `KEEPALIVE_TIMEOUT` from `gateway_extra_env` the way the image does + ## Tenant deployment Every resource the stack creates is named `${tenant}-litellm-${env}` (or diff --git a/terraform/litellm/gcp/cloudrun.tf b/terraform/litellm/gcp/cloudrun.tf index 405893b132b..78d4ffa2152 100644 --- a/terraform/litellm/gcp/cloudrun.tf +++ b/terraform/litellm/gcp/cloudrun.tf @@ -138,10 +138,16 @@ locals { "export DATABASE_URL_READ_REPLICA=\"postgresql://$${DATABASE_USER}:$${DATABASE_PASSWORD}@$${DATABASE_HOST_READ_REPLICA}:$${DATABASE_PORT_READ_REPLICA}/$${DATABASE_NAME}\"", ] + gateway_pool_env = var.gateway_connection_pool_enabled ? [ + { name = "LITELLM_PGBOUNCER_ENABLED", value = "true" }, + { name = "LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS", value = tostring(var.gateway_pool_max_db_connections) }, + { name = "LITELLM_PGBOUNCER_MAX_CLIENT_CONN", value = tostring(var.gateway_pool_max_client_conn) }, + ] : [] + gateway_uvicorn_args = "--host 0.0.0.0 --port 4000 --workers ${var.gateway_num_workers}" backend_uvicorn_args = "--host 0.0.0.0 --port 4001" - gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn gateway.main:app ${local.gateway_uvicorn_args};; *) exec uvicorn gateway.main:app ${local.gateway_uvicorn_args};; esac" + gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run python -m gateway.launch ${local.gateway_uvicorn_args};; *) exec python -m gateway.launch ${local.gateway_uvicorn_args};; esac" backend_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn backend.main:app ${local.backend_uvicorn_args};; *) exec uvicorn backend.main:app ${local.backend_uvicorn_args};; esac" gateway_args = join(" && ", concat( @@ -229,7 +235,7 @@ resource "google_cloud_run_v2_service" "gateway" { } dynamic "env" { - for_each = concat(local.shared_env_kv, local.gateway_otel_env_kv, local.billing_metrics_env_kv, local.gateway_extra_env_kv, local.proxy_config_env, local.metrics_env_kv) + for_each = concat(local.shared_env_kv, local.gateway_otel_env_kv, local.billing_metrics_env_kv, local.gateway_extra_env_kv, local.proxy_config_env, local.metrics_env_kv, local.gateway_pool_env) content { name = env.value.name value = env.value.value diff --git a/terraform/litellm/gcp/tests/connection_pool.tftest.hcl b/terraform/litellm/gcp/tests/connection_pool.tftest.hcl new file mode 100644 index 00000000000..439fe6b1d71 --- /dev/null +++ b/terraform/litellm/gcp/tests/connection_pool.tftest.hcl @@ -0,0 +1,135 @@ +# Plan-only coverage for the in-container PgBouncer knobs on the gateway +# service. `mock_provider` keeps this offline: no GCP credentials, no API +# calls, no resources. Run from terraform/litellm/gcp with `terraform test`. + +mock_provider "google" { + mock_resource "google_redis_instance" { + defaults = { + host = "10.0.0.4" + port = 6379 + server_ca_certs = [{ + cert = "-----BEGIN CERTIFICATE-----\nmock\n-----END CERTIFICATE-----" + }] + } + } +} + +mock_provider "google-beta" {} +mock_provider "random" {} + +variables { + project_id = "test-project" + tenant = "tenant" + env = "test" + allow_plaintext_lb = true + image_registry = "us-central1-docker.pkg.dev/test-project/litellm" +} + +run "pool_off_by_default" { + command = plan + + assert { + condition = length(local.gateway_pool_env) == 0 + error_message = "The gateway must get no LITELLM_PGBOUNCER_* env unless gateway_connection_pool_enabled is set." + } + + assert { + condition = !anytrue([ + for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : startswith(e.name, "LITELLM_PGBOUNCER_") + ]) + error_message = "The gateway service must carry no LITELLM_PGBOUNCER_* env by default." + } +} + +run "pool_enabled_renders_the_three_vars_with_configured_sizes" { + command = plan + + variables { + gateway_num_workers = 4 + gateway_connection_pool_enabled = true + gateway_pool_max_db_connections = 8 + gateway_pool_max_client_conn = 250 + } + + assert { + condition = alltrue([ + length(local.gateway_pool_env) == 3, + local.gateway_pool_env[0].name == "LITELLM_PGBOUNCER_ENABLED" && local.gateway_pool_env[0].value == "true", + local.gateway_pool_env[1].name == "LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS" && local.gateway_pool_env[1].value == "8", + local.gateway_pool_env[2].name == "LITELLM_PGBOUNCER_MAX_CLIENT_CONN" && local.gateway_pool_env[2].value == "250", + ]) + error_message = "The pool env must carry the enabled flag and the configured sizes as strings." + } + + assert { + condition = alltrue([ + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : e.name], "LITELLM_PGBOUNCER_ENABLED"), + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : e.name], "LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS"), + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : e.name], "LITELLM_PGBOUNCER_MAX_CLIENT_CONN"), + ]) + error_message = "The gateway service must receive all three LITELLM_PGBOUNCER_* env vars." + } + + assert { + condition = !anytrue(concat( + [for e in google_cloud_run_v2_service.backend[0].template[0].containers[0].env : startswith(e.name, "LITELLM_PGBOUNCER_")], + [for e in google_cloud_run_v2_job.migrations[0].template[0].template[0].containers[0].env : startswith(e.name, "LITELLM_PGBOUNCER_")], + )) + error_message = "The backend service and the migrations job must keep their direct database connection." + } +} + +run "pool_enabled_uses_the_module_default_sizes" { + command = plan + + variables { + gateway_connection_pool_enabled = true + } + + assert { + condition = alltrue([ + length(local.gateway_pool_env) == 3, + local.gateway_pool_env[1].name == "LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS" && local.gateway_pool_env[1].value == "20", + local.gateway_pool_env[2].name == "LITELLM_PGBOUNCER_MAX_CLIENT_CONN" && local.gateway_pool_env[2].value == "1000", + ]) + error_message = "The pool env must fall back to the module defaults of 20 upstream and 1000 client connections." + } +} + +run "gateway_starts_through_the_pool_aware_launcher" { + command = plan + + variables { + gateway_num_workers = 4 + } + + assert { + condition = alltrue([ + strcontains(local.gateway_launch_cmd, "exec python -m gateway.launch --host 0.0.0.0 --port 4000 --workers 4"), + strcontains(local.gateway_launch_cmd, "exec ddtrace-run python -m gateway.launch --host 0.0.0.0 --port 4000 --workers 4"), + !strcontains(local.gateway_launch_cmd, "uvicorn gateway.main:app"), + endswith(google_cloud_run_v2_service.gateway[0].template[0].containers[0].args[0], local.gateway_launch_cmd), + ]) + error_message = "The gateway must start through gateway.launch (with and without ddtrace) so the pooler starts once before uvicorn forks the workers." + } + + assert { + condition = strcontains(local.backend_launch_cmd, "uvicorn backend.main:app") + error_message = "The backend has no workers to share a pooler and keeps starting uvicorn directly." + } +} + +run "pool_sizes_below_one_fail_at_plan" { + command = plan + + variables { + gateway_connection_pool_enabled = true + gateway_pool_max_db_connections = 0 + gateway_pool_max_client_conn = 0 + } + + expect_failures = [ + var.gateway_pool_max_db_connections, + var.gateway_pool_max_client_conn, + ] +} diff --git a/terraform/litellm/gcp/variables.tf b/terraform/litellm/gcp/variables.tf index a753b4584a6..f298a0431c0 100644 --- a/terraform/litellm/gcp/variables.tf +++ b/terraform/litellm/gcp/variables.tf @@ -206,6 +206,45 @@ variable "gateway_num_workers" { } } +variable "gateway_connection_pool_enabled" { + description = <<-EOT + Run an in-container PgBouncer (transaction mode, loopback) in each gateway + instance, shared by every uvicorn worker. Without it each of the + `gateway_num_workers` workers opens its own Prisma pool straight to + Cloud SQL, so an instance's footprint against the database connection + ceiling is workers x connection_limit and grows with every instance. Sets + LITELLM_PGBOUNCER_ENABLED / LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS / + LITELLM_PGBOUNCER_MAX_CLIENT_CONN on the gateway container only. The + module's Cloud SQL authenticates with the static password in Secret + Manager, which is what the pooler needs. Mirrors the AWS stack's + gateway_connection_pool_enabled. + EOT + type = bool + default = false +} + +variable "gateway_pool_max_db_connections" { + description = "Upstream Cloud SQL connections one gateway instance may hold when gateway_connection_pool_enabled is set, regardless of gateway_num_workers. 20 suits 4 workers." + type = number + default = 20 + + validation { + condition = var.gateway_pool_max_db_connections >= 1 + error_message = "gateway_pool_max_db_connections must be >= 1." + } +} + +variable "gateway_pool_max_client_conn" { + description = "Client connections the in-container PgBouncer accepts from the gateway workers when gateway_connection_pool_enabled is set." + type = number + default = 1000 + + validation { + condition = var.gateway_pool_max_client_conn >= 1 + error_message = "gateway_pool_max_client_conn must be >= 1." + } +} + # Cloud Run autoscales out of the box (request-rate driven). The min/max # bounds mirror the HPA replica bounds in helm/litellm/values.yaml so each # stack scales over the same range. Cloud Run has no direct CPU-utilization diff --git a/tests/test_gateway/test_launch.py b/tests/test_gateway/test_launch.py new file mode 100644 index 00000000000..964a7021416 --- /dev/null +++ b/tests/test_gateway/test_launch.py @@ -0,0 +1,155 @@ +import os +import socket +import sys +import textwrap +import urllib.parse +from pathlib import Path +from typing import Final, cast + +import pytest +from uvicorn.importer import import_from_string +from uvicorn.main import main as uvicorn_main + +import gateway.main +from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv +from litellm.proxy.db.db_url_settings import DatabaseURLSettings +from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings + +DB_ENV: Final = { + "DATABASE_HOST": "db.internal", + "DATABASE_PORT": "5432", + "DATABASE_USER": "litellm_pool", + "DATABASE_NAME": "litellm", + "DATABASE_PASSWORD": "p@ss", +} + + +def _free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return cast(tuple[str, int], probe.getsockname())[1] + + +def _fake_pooler(tmp_path: Path) -> Path: + script: Final = tmp_path / "fake-pgbouncer" + script.write_text( + textwrap.dedent( + f"""\ + #!{sys.executable} + import configparser, select, socket, sys + if sys.argv[1:] == ["--version"]: + print("PgBouncer 1.25.2") + sys.exit(0) + ini = configparser.ConfigParser() + ini.read(sys.argv[1]) + port = ini.getint("pgbouncer", "listen_port") + tcp = socket.socket() + tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + tcp.bind(("127.0.0.1", port)) + tcp.listen() + unix = socket.socket(socket.AF_UNIX) + unix.bind(ini.get("pgbouncer", "unix_socket_dir") + f"/.s.PGSQL.{{port}}") + unix.listen() + while True: + for ready in select.select([tcp, unix], [], [])[0]: + ready.accept()[0].close() + """ + ) + ) + script.chmod(0o700) + return script + + +def _query(url: str) -> dict[str, str]: + return dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query)) + + +@pytest.fixture +def password_env(monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: + for var in ("DATABASE_URL", "IAM_TOKEN_DB_AUTH", "AZURE_POSTGRESQL_AUTH", "DATABASE_HOST_READ_REPLICA"): + monkeypatch.setenv(var, "") + monkeypatch.delenv(var) + for var, value in DB_ENV.items(): + monkeypatch.setenv(var, value) + return dict(DB_ENV) + + +def _uvicorn_params(argv: tuple[str, ...]) -> dict[str, object]: + return uvicorn_main.make_context("uvicorn", list(argv)).params + + +class TestUvicornArgv: + def test_keepalive_env_reaches_uvicorn(self): + params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"})) + assert params["app"] == GATEWAY_APP + assert params["workers"] == 4 + assert params["timeout_keep_alive"] == 75 + + def test_unset_env_keeps_the_uvicorn_default(self): + assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}))["timeout_keep_alive"] == 5 + + def test_an_explicit_flag_wins_over_the_env(self): + argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}) + assert _uvicorn_params(argv)["timeout_keep_alive"] == 30 + + def test_the_app_uvicorn_is_told_to_serve_is_the_trimmed_gateway(self): + assert import_from_string(cast(str, _uvicorn_params(uvicorn_argv((), {}))["app"])) is gateway.main.app + + +class TestPoolDatabaseUrl: + def test_a_disabled_pooler_yields_no_url_to_install(self, password_env: dict[str, str]): + settings: Final = DatabaseURLSettings.from_env() + settings.apply_to_env() + environ: Final = {"DATABASE_URL": "postgresql://litellm_pool:p%40ss@db.internal:5432/litellm"} + assert pool_database_url(settings, PgBouncerSettings(enabled=False), environ) is None + + def test_a_missing_upstream_url_is_reported(self, password_env: dict[str, str]): + environ: Final[dict[str, str]] = {} + outcome: Final = pool_database_url(DatabaseURLSettings.from_env(), PgBouncerSettings(enabled=True), environ) + assert isinstance(outcome, PgBouncerError) + assert "DATABASE_URL" in outcome.reason + + def test_token_auth_is_refused(self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + environ: Final = {"DATABASE_URL": "postgresql://litellm:token@db.internal:5432/litellm"} + outcome: Final = pool_database_url( + DatabaseURLSettings.from_env(), + PgBouncerSettings(enabled=True, port=_free_port(), binary=str(_fake_pooler(tmp_path))), + environ, + ) + assert isinstance(outcome, PgBouncerError) + assert "IAM_TOKEN_DB_AUTH" in outcome.reason + + +class TestMain: + def test_workers_inherit_the_loopback_url_the_supervisor_installed( + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ): + port: Final = _free_port() + monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true") + monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", str(port)) + monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(tmp_path))) + monkeypatch.setenv("KEEPALIVE_TIMEOUT", "75") + served: Final[list[tuple[str, ...]]] = [] + main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv))) + + pooled: Final = os.environ["DATABASE_URL"] + assert urllib.parse.urlsplit(pooled).netloc == f"litellm_pool:p%40ss@127.0.0.1:{port}" + assert _query(pooled)["pgbouncer"] == "true" + assert _uvicorn_params(served[0])["timeout_keep_alive"] == 75 + + DatabaseURLSettings.from_env().apply_to_env() + worker_url: Final = os.environ["DATABASE_URL"] + assert urllib.parse.urlsplit(worker_url).netloc == f"litellm_pool:p%40ss@127.0.0.1:{port}" + assert _query(worker_url)["pgbouncer"] == "true" + + def test_a_pooler_that_cannot_start_stops_the_gateway_before_uvicorn( + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ): + monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true") + monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(tmp_path / "missing-pgbouncer")) + served: Final[list[tuple[str, ...]]] = [] + with pytest.raises(SystemExit) as stopped: + main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv))) + assert "missing-pgbouncer" in str(stopped.value) + assert served == [] diff --git a/tests/test_litellm/proxy/db/test_pgbouncer.py b/tests/test_litellm/proxy/db/test_pgbouncer.py new file mode 100644 index 00000000000..da1d4c91f09 --- /dev/null +++ b/tests/test_litellm/proxy/db/test_pgbouncer.py @@ -0,0 +1,624 @@ +import configparser +import logging +import os +import signal +import socket +import stat +import sys +import tempfile +import textwrap +import time +import urllib.parse +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final, cast + +import pytest + +from litellm._logging import verbose_proxy_logger +from litellm.proxy.db.pgbouncer import ( + PgBouncerError, + PgBouncerPlan, + PgBouncerProcess, + PgBouncerSettings, + pgbouncer_version, + plan_pgbouncer, + start_in_container_pgbouncer, + unix_socket_path, + write_pgbouncer_files, +) + +UPSTREAM: Final = ( + "postgresql://app:p%40ss%27w@db.internal:5433/litellm" + "?schema=public&connection_limit=10&pool_timeout=20" + "&sslmode=require&sslaccept=strict&sslcert=/certs/ca.pem" + "&options=-c%20statement_timeout%3D7000%20-c%20lock_timeout%3D3000" +) +SETTINGS: Final = PgBouncerSettings(enabled=True, port=6543, max_db_connections=8, max_client_conn=400) + + +def _plan(url: str = UPSTREAM, run_as_user: str | None = None) -> PgBouncerPlan: + plan: Final = plan_pgbouncer(url, SETTINGS, Path("/run/pgb"), run_as_user) + assert isinstance(plan, PgBouncerPlan), plan + return plan + + +def _ini(plan: PgBouncerPlan) -> configparser.ConfigParser: + parser: Final = configparser.ConfigParser(interpolation=None) + parser.read_string(plan.ini) + return parser + + +def _query(url: str) -> dict[str, str]: + return dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query, keep_blank_values=True)) + + +class TestPlanPgBouncer: + def test_upstream_credentials_and_timeouts_move_into_the_pgbouncer_config(self): + ini: Final = _ini(_plan()) + assert ini["databases"]["litellm"] == ( + "host='db.internal' port=5433 dbname='litellm' user='app' password='p@ss''w' " + "connect_query='SET statement_timeout TO ''7000''; SET lock_timeout TO ''3000'''" + ) + assert _plan().userlist == '"app" "p@ss\'w"\n' + + def test_an_upstream_without_a_port_is_reached_on_the_postgres_default(self): + ini: Final = _ini(_plan("postgresql://app:pw@db/litellm")) + assert ini["databases"]["litellm"] == "host='db' port=5432 dbname='litellm' user='app' password='pw'" + + def test_pool_is_sized_from_settings_in_transaction_mode(self): + pgb: Final = _ini(_plan())["pgbouncer"] + assert pgb["pool_mode"] == "transaction" + assert pgb["max_db_connections"] == "8" + assert pgb["default_pool_size"] == "8" + assert pgb["max_client_conn"] == "400" + assert pgb["auth_type"] == "scram-sha-256" + assert pgb["listen_addr"] == "127.0.0.1" + assert pgb["listen_port"] == "6543" + assert pgb["auth_file"] == "/run/pgb/userlist.txt" + assert pgb["unix_socket_dir"] == "/run/pgb" + + def test_the_app_user_can_read_the_pgbouncer_console(self): + assert _ini(_plan())["pgbouncer"]["stats_users"] == "app" + + @pytest.mark.parametrize("user", ["app,admin", "app%20admin", "app%09admin"]) + def test_a_user_pgbouncer_would_split_into_several_console_users_is_refused(self, user: str): + outcome: Final = plan_pgbouncer(f"postgresql://{user}:pw@db/litellm", SETTINGS, Path("/run/pgb"), None) + assert isinstance(outcome, PgBouncerError) + assert "stats_users" in outcome.reason + + def test_pooled_url_points_prisma_at_loopback_without_prepared_statements(self): + pooled: Final = urllib.parse.urlsplit(_plan().pooled_url) + assert (pooled.hostname, pooled.port, pooled.path) == ("127.0.0.1", 6543, "/litellm") + assert (pooled.username, pooled.password) == ("app", "p%40ss%27w") + assert _query(_plan().pooled_url) == { + "schema": "public", + "connection_limit": "10", + "pool_timeout": "20", + "pgbouncer": "true", + } + + @pytest.mark.parametrize("hop_param", ["channel_binding=require", "gssencmode=require"]) + def test_transport_params_for_the_postgres_hop_stay_off_the_plain_tcp_loopback_url(self, hop_param: str): + pooled: Final = _plan(f"postgresql://app:pw@db/litellm?connection_limit=5&{hop_param}").pooled_url + assert _query(pooled) == {"connection_limit": "5", "pgbouncer": "true"} + + def test_verified_tls_becomes_server_side_verify_full_with_a_ca_copy_in_the_runtime_dir(self): + plan: Final = _plan() + pgb: Final = _ini(plan)["pgbouncer"] + assert pgb["server_tls_sslmode"] == "verify-full" + assert pgb["server_tls_ca_file"] == "/run/pgb/server-ca.pem" + assert plan.ca_source == "/certs/ca.pem" + + def test_unverified_require_stays_require_without_a_ca_file(self): + plan: Final = _plan("postgresql://app:pw@db/litellm?sslmode=require") + pgb: Final = _ini(plan)["pgbouncer"] + assert pgb["server_tls_sslmode"] == "require" + assert "server_tls_ca_file" not in pgb + assert plan.ca_source is None + + def test_no_tls_params_default_to_prefer(self): + assert _ini(_plan("postgresql://app:pw@db/litellm"))["pgbouncer"]["server_tls_sslmode"] == "prefer" + + def test_verification_without_a_ca_bundle_is_refused(self): + outcome: Final = plan_pgbouncer( + "postgresql://app:pw@db/litellm?sslmode=require&sslaccept=strict", SETTINGS, Path("/run/pgb"), None + ) + assert isinstance(outcome, PgBouncerError) + assert "sslcert" in outcome.reason + + def test_client_certificates_are_refused(self): + outcome: Final = plan_pgbouncer( + "postgresql://app:pw@db/litellm?sslidentity=/certs/client.p12", SETTINGS, Path("/run/pgb"), None + ) + assert isinstance(outcome, PgBouncerError) + assert "sslidentity" in outcome.reason + + @pytest.mark.parametrize( + "url", + [ + "postgresql://app@db/litellm", + "postgresql://app:pw@db", + "postgresql://:pw@db/litellm", + ], + ) + def test_urls_missing_forwardable_credentials_are_refused(self, url: str): + outcome: Final = plan_pgbouncer(url, SETTINGS, Path("/run/pgb"), None) + assert isinstance(outcome, PgBouncerError) + + def test_every_options_spelling_becomes_a_set_statement(self): + options: Final = urllib.parse.quote("-c a=1 -cb=2 --c=3") + ini: Final = _ini(_plan(f"postgresql://app:pw@db/litellm?options={options}")) + assert ini["databases"]["litellm"].endswith("connect_query='SET a TO ''1''; SET b TO ''2''; SET c TO ''3'''") + + def test_options_that_are_not_settings_are_refused(self): + outcome: Final = plan_pgbouncer( + "postgresql://app:pw@db/litellm?options=-c%20search_path", SETTINGS, Path("/run/pgb"), None + ) + assert isinstance(outcome, PgBouncerError) + assert "options" in outcome.reason + + def test_run_as_user_is_only_written_when_given(self): + assert _ini(_plan(run_as_user="nobody"))["pgbouncer"]["user"] == "nobody" + assert "user" not in _ini(_plan())["pgbouncer"] + + +class TestWritePgBouncerFiles: + def test_files_hold_the_plan_and_are_private_to_the_owner(self, tmp_path: Path): + plan: Final = _plan("postgresql://app:pw@db/litellm") + ini_path: Final = write_pgbouncer_files(plan, tmp_path, None) + assert isinstance(ini_path, Path), ini_path + assert ini_path == tmp_path / "pgbouncer.ini" + assert ini_path.read_text() == plan.ini + assert (tmp_path / "userlist.txt").read_text() == plan.userlist + for path in (ini_path, tmp_path / "userlist.txt"): + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert not (tmp_path / "server-ca.pem").exists() + + def test_the_ca_bundle_is_copied_next_to_the_ini_pgbouncer_reads(self, tmp_path: Path): + bundle: Final = tmp_path / "rds-root.pem" + bundle.write_text("-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n") + runtime_dir: Final = tmp_path / "run" + runtime_dir.mkdir() + plan: Final = plan_pgbouncer( + f"postgresql://app:pw@db/litellm?sslmode=verify-full&sslcert={bundle}", SETTINGS, runtime_dir, None + ) + assert isinstance(plan, PgBouncerPlan), plan + ini_path: Final = write_pgbouncer_files(plan, runtime_dir, None) + assert isinstance(ini_path, Path), ini_path + ca_file: Final = Path(_ini(plan)["pgbouncer"]["server_tls_ca_file"]) + assert ca_file.parent == runtime_dir + assert ca_file.read_text() == bundle.read_text() + + def test_an_unreadable_ca_bundle_is_reported(self, tmp_path: Path): + plan: Final = plan_pgbouncer( + f"postgresql://app:pw@db/litellm?sslmode=verify-full&sslcert={tmp_path / 'missing.pem'}", + SETTINGS, + tmp_path, + None, + ) + assert isinstance(plan, PgBouncerPlan), plan + outcome: Final = write_pgbouncer_files(plan, tmp_path, None) + assert isinstance(outcome, PgBouncerError) + assert "missing.pem" in outcome.reason + assert not (tmp_path / "pgbouncer.ini").exists() + + +def _bound_port(sock: socket.socket) -> int: + return cast(tuple[str, int], sock.getsockname())[1] + + +def _free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return _bound_port(probe) + + +def _fake_pooler( + tmp_path: Path, + port: int, + exit_immediately: bool = False, + port_file: Path | None = None, + bind_delay_seconds: float = 0.0, + version_banner: str = "PgBouncer 1.25.2\nlibevent 2.1.13-stable", +) -> Path: + """An executable that listens like PgBouncer: on the TCP port first, then on ``.s.PGSQL.`` in the socket dir. + + Port and socket dir come from the ini it is given, else from ``port`` and + ``tmp_path``. With ``port_file`` each start reads the port from that file + instead. ``bind_delay_seconds`` holds the bind back, like a slow start. + ``--version`` prints ``version_banner``. + """ + script: Final = tmp_path / "fake-pgbouncer" + script.write_text( + textwrap.dedent( + f"""\ + #!{sys.executable} + import configparser, os, pathlib, select, socket, sys, time + if sys.argv[1:] == ["--version"]: + print({version_banner!r}) + sys.exit(0) + if {exit_immediately!r}: + sys.exit(3) + ini = configparser.ConfigParser() + ini.read(sys.argv[1:2]) + port = ini.getint("pgbouncer", "listen_port", fallback={port}) + if not {port_file is None!r}: + port = int(pathlib.Path({str(port_file)!r}).read_text()) + socket_dir = ini.get("pgbouncer", "unix_socket_dir", fallback={str(tmp_path)!r}) + socket_path = f"{{socket_dir}}/.s.PGSQL.{{port}}" + time.sleep({bind_delay_seconds!r}) + listener = socket.socket() + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", port)) + listener.listen() + if os.path.exists(socket_path): + os.unlink(socket_path) + unix_listener = socket.socket(socket.AF_UNIX) + unix_listener.bind(socket_path) + unix_listener.listen() + while True: + for ready in select.select([listener, unix_listener], [], [])[0]: + conn, _ = ready.accept() + conn.close() + """ + ) + ) + script.chmod(0o700) + return script + + +def _listening(port: int) -> bool: + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.5): + return True + except OSError: + return False + + +def _wait_until(condition: Callable[[], bool], timeout_seconds: float = 5.0) -> bool: + deadline: Final = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if condition(): + return True + time.sleep(0.05) + return False + + +class TestPgBouncerProcess: + def test_start_waits_for_the_listener_and_stop_ends_it(self, tmp_path: Path): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port)),), port=port, socket_path=unix_socket_path(tmp_path, port) + ) + assert pooler.start() is None + assert _listening(port) + pid: Final = pooler.pid + assert pid is not None + pooler.stop() + assert _wait_until(lambda: not _listening(port)) + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + + def test_a_crashed_pooler_is_restarted_with_a_new_pid(self, tmp_path: Path): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port)),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + restart_delay_seconds=0.1, + ) + assert pooler.start() is None + first_pid: Final = pooler.pid + assert first_pid is not None + os.kill(first_pid, signal.SIGKILL) + assert _wait_until(lambda: pooler.pid not in (None, first_pid) and _listening(port)) + pooler.stop() + assert _wait_until(lambda: not _listening(port)) + + def test_a_failed_restart_is_retried_until_the_pooler_is_back( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ): + port: Final = _free_port() + script: Final = _fake_pooler(tmp_path, port) + pooler: Final = PgBouncerProcess( + argv=(str(script),), port=port, socket_path=unix_socket_path(tmp_path, port), restart_delay_seconds=0.1 + ) + assert pooler.start() is None + first_pid: Final = pooler.pid + assert first_pid is not None + hidden: Final = script.rename(tmp_path / "hidden") + with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): + os.kill(first_pid, signal.SIGKILL) + assert _wait_until(lambda: any("could not be restarted" in record.message for record in caplog.records)) + assert not _listening(port) + hidden.rename(script) + assert _wait_until(lambda: pooler.pid not in (None, first_pid) and _listening(port)) + pooler.stop() + assert _wait_until(lambda: not _listening(port)) + + def test_a_replacement_that_never_listens_is_replaced_again(self, tmp_path: Path, caplog: pytest.LogCaptureFixture): + port: Final = _free_port() + port_file: Final = tmp_path / "port" + port_file.write_text(str(port)) + script: Final = _fake_pooler(tmp_path, port, port_file=port_file) + pooler: Final = PgBouncerProcess( + argv=(str(script),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + restart_delay_seconds=0.1, + ready_timeout_seconds=0.3, + ) + assert pooler.start() is None + first_pid: Final = pooler.pid + assert first_pid is not None + wrong_port: Final = _free_port() + port_file.write_text(str(wrong_port)) + with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): + os.kill(first_pid, signal.SIGKILL) + assert _wait_until(lambda: _listening(wrong_port)) + assert _wait_until(lambda: any("did not start listening" in record.message for record in caplog.records)) + port_file.write_text(str(port)) + assert _wait_until(lambda: _listening(port)) + assert _wait_until(lambda: not _listening(wrong_port)) + pooler.stop() + assert _wait_until(lambda: not _listening(port)) + + def test_stopping_during_the_restart_delay_leaves_no_pooler_behind(self, tmp_path: Path): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port)),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + restart_delay_seconds=0.3, + ) + assert pooler.start() is None + first_pid: Final = pooler.pid + assert first_pid is not None + os.kill(first_pid, signal.SIGKILL) + assert _wait_until(lambda: not _listening(port)) + pooler.stop() + time.sleep(1.0) + assert not _listening(port) + assert pooler.pid == first_pid + + def test_a_stopped_pooler_is_not_restarted(self, tmp_path: Path, caplog: pytest.LogCaptureFixture): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port)),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + restart_delay_seconds=0.1, + ) + assert pooler.start() is None + with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): + pooler.stop() + time.sleep(0.5) + assert not _listening(port) + assert caplog.records == [] + + def test_a_pooler_that_exits_during_startup_is_reported(self, tmp_path: Path): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port, exit_immediately=True)),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + ) + outcome: Final = pooler.start() + assert isinstance(outcome, PgBouncerError) + assert "status 3" in outcome.reason + + def test_a_missing_binary_is_reported(self, tmp_path: Path): + outcome: Final = PgBouncerProcess( + argv=("/nonexistent/pgbouncer",), port=_free_port(), socket_path=tmp_path / "sock" + ).start() + assert isinstance(outcome, PgBouncerError) + assert "/nonexistent/pgbouncer" in outcome.reason + + def test_a_port_owned_by_someone_else_is_refused_before_spawning(self, tmp_path: Path): + with socket.socket() as squatter: + squatter.bind(("127.0.0.1", 0)) + squatter.listen() + port: Final = _bound_port(squatter) + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port)),), port=port, socket_path=unix_socket_path(tmp_path, port) + ) + outcome: Final = pooler.start() + assert isinstance(outcome, PgBouncerError) + assert f"127.0.0.1:{port} is already in use" in outcome.reason + assert pooler.pid is None + + def test_a_replacement_waits_until_a_squatter_leaves_the_port( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port)),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + restart_delay_seconds=0.5, + ) + assert pooler.start() is None + first_pid: Final = pooler.pid + assert first_pid is not None + os.kill(first_pid, signal.SIGKILL) + assert _wait_until(lambda: not _listening(port)) + with socket.socket() as squatter, caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): + squatter.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + squatter.bind(("127.0.0.1", port)) + squatter.listen() + assert _wait_until(lambda: any("already in use" in record.message for record in caplog.records)) + assert pooler.pid == first_pid + assert _wait_until(lambda: pooler.pid not in (None, first_pid) and _listening(port)) + pooler.stop() + assert _wait_until(lambda: not _listening(port)) + + def test_a_listener_that_grabs_the_port_after_the_spawn_is_not_taken_for_the_pooler(self, tmp_path: Path): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port, bind_delay_seconds=0.5)),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + ready_timeout_seconds=3.0, + ) + with socket.socket() as squatter, ThreadPoolExecutor(max_workers=1) as starter: + squatter.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + starting: Final = starter.submit(pooler.start) + assert _wait_until(lambda: pooler.pid is not None) + squatter.bind(("127.0.0.1", port)) + squatter.listen() + outcome: Final = starting.result() + assert isinstance(outcome, PgBouncerError) + assert "exited with status 1" in outcome.reason + + def test_a_port_served_by_a_stranger_while_the_pooler_is_still_starting_is_reported(self, tmp_path: Path): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port, bind_delay_seconds=30.0)),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + ready_timeout_seconds=0.5, + ) + with socket.socket() as squatter, ThreadPoolExecutor(max_workers=1) as starter: + squatter.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + starting: Final = starter.submit(pooler.start) + assert _wait_until(lambda: pooler.pid is not None) + squatter.bind(("127.0.0.1", port)) + squatter.listen() + outcome: Final = starting.result() + assert isinstance(outcome, PgBouncerError) + assert f"127.0.0.1:{port} is served by another process" in outcome.reason + pid: Final = pooler.pid + assert pid is not None + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + + def test_a_pooler_that_never_listens_times_out(self, tmp_path: Path): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, _free_port())),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + ready_timeout_seconds=0.5, + ) + outcome: Final = pooler.start() + assert isinstance(outcome, PgBouncerError) + assert "did not start listening" in outcome.reason + pid: Final = pooler.pid + assert pid is not None + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + + +def _runtime_dir_listening_on(port: int) -> Path: + matches: Final = tuple( + ini.parent + for ini in Path(tempfile.gettempdir()).glob("litellm-pgbouncer-*/pgbouncer.ini") + if f"listen_port = {port}\n" in ini.read_text() + ) + assert len(matches) == 1, matches + return matches[0] + + +class TestStartInContainerPgBouncer: + def test_returns_the_loopback_url_once_the_pooler_listens(self, tmp_path: Path): + port: Final = _free_port() + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) + pooled: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm?connection_limit=5") + assert pooled == f"postgresql://app:pw@127.0.0.1:{port}/litellm?connection_limit=5&pgbouncer=true" + assert _listening(port) + + @pytest.mark.filterwarnings("ignore:This process .* is multi-threaded:DeprecationWarning") + def test_a_forked_worker_exiting_leaves_the_pooler_and_its_files_to_the_parent(self, tmp_path: Path): + port: Final = _free_port() + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) + exit_hooks: Final[list[Callable[[], None]]] = [] + pooled: Final = start_in_container_pgbouncer( + settings, "postgresql://app:pw@db/litellm", register_exit_hook=exit_hooks.append + ) + assert isinstance(pooled, str), pooled + runtime_dir: Final = _runtime_dir_listening_on(port) + + worker: Final = os.fork() + if worker == 0: + try: + for hook in exit_hooks: + hook() + finally: + os._exit(0) + if not _wait_until(lambda: os.waitpid(worker, os.WNOHANG) != (0, 0)): + os.kill(worker, signal.SIGKILL) + pytest.fail("the forked worker did not exit: an exit hook blocked on state inherited from the parent") + assert _listening(port) + assert (runtime_dir / "pgbouncer.ini").exists() + + for hook in exit_hooks: + hook() + assert _wait_until(lambda: not _listening(port)) + assert not runtime_dir.exists() + + def test_a_bad_upstream_url_is_reported_without_starting_anything(self, tmp_path: Path): + port: Final = _free_port() + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) + outcome: Final = start_in_container_pgbouncer(settings, "postgresql://app@db/litellm") + assert isinstance(outcome, PgBouncerError) + assert not _listening(port) + + def test_token_auth_is_refused_without_starting_anything(self, tmp_path: Path): + port: Final = _free_port() + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) + outcome: Final = start_in_container_pgbouncer( + settings, "postgresql://app:pw@db/litellm", token_auth_enabled=True + ) + assert isinstance(outcome, PgBouncerError) + assert "IAM_TOKEN_DB_AUTH" in outcome.reason + assert "AZURE_POSTGRESQL_AUTH" in outcome.reason + assert not _listening(port) + + def test_a_pgbouncer_that_survives_a_failed_tcp_bind_is_refused_without_starting(self, tmp_path: Path): + port: Final = _free_port() + binary: Final = _fake_pooler(tmp_path, port, version_banner="PgBouncer 1.18.1\nlibevent 2.1.12-stable") + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) + outcome: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm") + assert isinstance(outcome, PgBouncerError) + assert "PgBouncer 1.18" in outcome.reason + assert "1.19" in outcome.reason + assert not _listening(port) + + def test_the_first_version_that_dies_on_a_failed_tcp_bind_is_accepted(self, tmp_path: Path): + port: Final = _free_port() + binary: Final = _fake_pooler(tmp_path, port, version_banner="PgBouncer 1.19.0") + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) + assert start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm") == ( + f"postgresql://app:pw@127.0.0.1:{port}/litellm?pgbouncer=true" + ) + assert _listening(port) + + +class TestPgBouncerVersion: + def test_reads_major_and_minor_from_the_banner(self, tmp_path: Path): + assert pgbouncer_version(str(_fake_pooler(tmp_path, _free_port()))) == (1, 25) + + def test_a_binary_that_cannot_run_is_reported(self, tmp_path: Path): + outcome: Final = pgbouncer_version(str(tmp_path / "missing-pgbouncer")) + assert isinstance(outcome, PgBouncerError) + assert "missing-pgbouncer" in outcome.reason + + def test_a_banner_without_a_version_is_reported(self, tmp_path: Path): + outcome: Final = pgbouncer_version(str(_fake_pooler(tmp_path, _free_port(), version_banner="something else"))) + assert isinstance(outcome, PgBouncerError) + assert "something else" in outcome.reason + + +class TestPgBouncerSettings: + def test_reads_the_litellm_pgbouncer_env_vars(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true") + monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", "7000") + monkeypatch.setenv("LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS", "12") + settings: Final = PgBouncerSettings() + assert (settings.enabled, settings.port, settings.max_db_connections) == (True, 7000, 12) + + def test_defaults_are_off(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False) + assert PgBouncerSettings().enabled is False diff --git a/tests/test_litellm/test_component_entrypoint.py b/tests/test_litellm/test_component_entrypoint.py index 09837d2b233..0c2a533b8bc 100644 --- a/tests/test_litellm/test_component_entrypoint.py +++ b/tests/test_litellm/test_component_entrypoint.py @@ -38,12 +38,16 @@ _STUB_TEMPLATE = """#!/bin/sh _ENTRYPOINT_RE = re.compile(r"^ENTRYPOINT\s+(\[.*\])\s*$", re.MULTILINE) _CMD_RE = re.compile(r"^CMD\s+(\[.*\])\s*$", re.MULTILINE) _COPY_RE = re.compile(r"^COPY\s+(?!--from)(\S+)\s+(\S+)\s*$", re.MULTILINE) -_APP_TARGET_RE = re.compile(r"(?:gateway|backend)\.main:app") +_APP_TARGET_RE = re.compile(r"(?:gateway|backend)\.main:app|gateway\.launch") _TF_STRING_LOCAL_RE = re.compile(r'^\s*(\w+)\s*=\s*"((?:[^"\\]|\\.)*)"\s*$', re.MULTILINE) _TF_INTERPOLATION_RE = re.compile(r"\$\{(local|var)\.(\w+)\}") TERRAFORM_LAUNCH_SITES = {TERRAFORM_ECS: 2, TERRAFORM_CLOUDRUN: 2} TERRAFORM_VAR_STUBS = {"gateway_num_workers": "2"} +COMPONENT_LAUNCHERS = { + "gateway": ("python", "-m", "gateway.launch"), + "backend": ("uvicorn", "backend.main:app"), +} _MAX_INTERPOLATION_PASSES = 5 @@ -63,7 +67,7 @@ def _run_entrypoint( """Run `script` with stubbed executables on PATH and return the recorded lines.""" bin_dir = tmp_path / "bin" bin_dir.mkdir(parents=True) - _write_stubs(bin_dir, ("ddtrace-run", "uvicorn", "litellm")) + _write_stubs(bin_dir, ("ddtrace-run", "uvicorn", "python", "litellm")) record = tmp_path / "record.txt" env = { @@ -330,22 +334,63 @@ def test_entrypoint_script_has_no_carriage_returns() -> None: @pytest.mark.parametrize( - "dockerfile, app_target", + "dockerfile, launcher", [ - (GATEWAY_DOCKERFILE, "gateway.main:app"), - (BACKEND_DOCKERFILE, "backend.main:app"), + (GATEWAY_DOCKERFILE, "python -m gateway.launch"), + (BACKEND_DOCKERFILE, "uvicorn backend.main:app"), ], ) -def test_component_images_launch_uvicorn_through_the_entrypoint(dockerfile: Path, app_target: str) -> None: +def test_component_images_launch_uvicorn_through_the_entrypoint(dockerfile: Path, launcher: str) -> None: entrypoint = " ".join(_entrypoint_argv(dockerfile)) assert IMAGE_ENTRYPOINT_PATH in entrypoint, f"{dockerfile} bypasses the ddtrace-aware entrypoint" - assert app_target in entrypoint - assert entrypoint.index(IMAGE_ENTRYPOINT_PATH) < entrypoint.index("uvicorn"), ( + assert launcher in entrypoint + assert entrypoint.index(IMAGE_ENTRYPOINT_PATH) < entrypoint.index(launcher), ( f"{dockerfile} must invoke uvicorn through the entrypoint, not the other way around" ) +@pytest.mark.parametrize( + "use_ddtrace, num_workers, expected_exec, expected_args", + [ + (None, "4", "exec=python", "args=-m gateway.launch --workers 4 --host 0.0.0.0 --port 4000"), + (None, None, "exec=python", "args=-m gateway.launch --workers 1 --host 0.0.0.0 --port 4000"), + ("true", "4", "exec=ddtrace-run", "args=python -m gateway.launch --workers 4 --host 0.0.0.0 --port 4000"), + ], +) +def test_gateway_image_execs_the_supervisor_with_its_worker_count( + use_ddtrace: str | None, num_workers: str | None, expected_exec: str, expected_args: str, tmp_path: Path +) -> None: + """Run the gateway image's ENTRYPOINT + CMD and record what the container execs. + + The Dockerfile's `/app/...` script path is resolved to the checked-in script and `python` + is stubbed on PATH, so the assertion is on the argv `gateway.launch` receives, not on the + Dockerfile text. + """ + entrypoint = tuple( + part.replace(IMAGE_ENTRYPOINT_PATH, str(COMPONENT_ENTRYPOINT)) for part in _entrypoint_argv(GATEWAY_DOCKERFILE) + ) + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + _write_stubs(bin_dir, ("ddtrace-run", "python", "uvicorn")) + record = tmp_path / "record.txt" + overrides = {"USE_DDTRACE": use_ddtrace, "NUM_WORKERS": num_workers} + env = { + **{k: v for k, v in os.environ.items() if k not in ("DD_TRACE_OPENAI_ENABLED", *overrides)}, + **{k: v for k, v in overrides.items() if v is not None}, + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "RECORD": str(record), + "PYTHONPATH": PYTHONPATH_SENTINEL, + } + + result = subprocess.run( + [*entrypoint, *_cmd_argv(GATEWAY_DOCKERFILE)], env=env, capture_output=True, text=True, check=False + ) + + assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" + assert tuple(record.read_text().splitlines())[:2] == (expected_exec, expected_args) + + @pytest.mark.parametrize("dockerfile", [GATEWAY_DOCKERFILE, BACKEND_DOCKERFILE]) def test_component_images_make_the_entrypoint_executable(dockerfile: Path) -> None: body = dockerfile.read_text() @@ -367,17 +412,18 @@ def test_terraform_launch_command_matches_the_script_contract( implementations under the same environment and asserts they agree on which binary is exec'd and on whether the openai integration is disabled. """ - app_target = f"{component}.main:app" + launcher = COMPONENT_LAUNCHERS[component] + app_target = " ".join(launcher[1:]) command = _resolve_tf_local(terraform_file, f"{component}_launch_cmd") bin_dir = tmp_path / "bin" bin_dir.mkdir(parents=True) - _write_stubs(bin_dir, ("ddtrace-run", "uvicorn")) + _write_stubs(bin_dir, ("ddtrace-run", "uvicorn", "python")) from_terraform = _run_shell_command(command, bin_dir, tmp_path / "terraform.txt", use_ddtrace) from_script = _run_entrypoint( COMPONENT_ENTRYPOINT, - ("uvicorn", app_target), + launcher, use_ddtrace=use_ddtrace, tmp_path=tmp_path / "script", ) @@ -387,12 +433,14 @@ def test_terraform_launch_command_matches_the_script_contract( ) assert from_terraform[2] == from_script[2], f"{terraform_file} disagrees with the script on the openai integration" assert app_target in from_terraform[1] + assert "gateway.main:app" not in from_terraform[1], f"{terraform_file} bypasses the gateway.launch supervisor" if use_ddtrace in TRUTHY_USE_DDTRACE: assert from_terraform[0] == "exec=ddtrace-run" + assert from_terraform[1].startswith(f"args={launcher[0]} ") assert from_terraform[2] == "DD_TRACE_OPENAI_ENABLED=False" else: - assert from_terraform[0] == "exec=uvicorn" + assert from_terraform[0] == f"exec={launcher[0]}" assert from_terraform[2] == "DD_TRACE_OPENAI_ENABLED=" From 7658cd53aaf5f8cc217b9a9a25891851eaa64cbc Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 10 Sep 2026 22:25:03 +0000 Subject: [PATCH 088/126] fix(caching): judge swallowed Redis failures per admission and throttle worker tracebacks per error type Swallowed failures are now reported to the breaker when the admitted call exits, so a call admitted before the breaker opened cannot refresh the open timer or knock out the recovery probe. The sync batch read raises the typed open-breaker error like its async twin so DualCache releases its batch reservations, and LoggingWorker throttles tracebacks per exception class instead of worker-wide Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/dual_cache.py | 3 + litellm/caching/redis_cache.py | 60 ++++++++++------- litellm/litellm_core_utils/logging_worker.py | 20 +++--- tests/test_litellm/caching/test_dual_cache.py | 20 ++++++ .../test_litellm/caching/test_redis_cache.py | 65 +++++++++++++++++-- .../litellm_core_utils/test_logging_worker.py | 32 ++++++++- 6 files changed, 162 insertions(+), 38 deletions(-) diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index 6fb918ce748..5dd0493cd65 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -206,6 +206,9 @@ class DualCache(BaseCache): redis_result: Final = self.redis_cache.batch_get_cache( key_list=sublist_keys, parent_otel_span=parent_otel_span ) + except RedisCircuitBreakerOpenError: + self._rollback_redis_batch_key_reservations(previous_access_times) + return result except Exception: # Do not throttle subsequent callers if the Redis read fails. self._rollback_redis_batch_key_reservations(previous_access_times) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 6c19b018c1d..3ecf805ed0e 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -285,7 +285,9 @@ class RedisCircuitBreaker: _RedisCallResult = TypeVar("_RedisCallResult") -_swallowed_redis_failures: Final[ContextVar[int]] = ContextVar("litellm_swallowed_redis_failures", default=0) +_swallowed_redis_failures: Final[ContextVar[tuple[bool, ...]]] = ContextVar( + "litellm_swallowed_redis_failures", default=() +) def _opaque_kwarg_key(value: object) -> str: @@ -405,8 +407,8 @@ def _breaker_metrics() -> _BreakerMetrics: return _BreakerMetrics() -def _record_swallowed_redis_failure(breaker: RedisCircuitBreaker, exc: BaseException) -> None: - """Record a Redis failure that the calling method is about to swallow. +def _record_swallowed_redis_failure(exc: BaseException) -> None: + """Note a Redis failure that the calling method is about to swallow, for the breaker exit to judge. The marker is a ContextVar rather than a counter on the breaker because breakers are shared by every concurrent caller. A plain shared counter cannot tell "my call failed" @@ -416,8 +418,7 @@ def _record_swallowed_redis_failure(breaker: RedisCircuitBreaker, exc: BaseExcep """ if not _is_redis_health_failure(exc): return - breaker.record_failure(is_timeout=_is_redis_timeout_failure(exc)) - _swallowed_redis_failures.set(_swallowed_redis_failures.get() + 1) + _swallowed_redis_failures.set((*_swallowed_redis_failures.get(), _is_redis_timeout_failure(exc))) @dataclass(frozen=True, slots=True) @@ -430,25 +431,42 @@ def _enter_circuit_breaker(breaker: RedisCircuitBreaker, name: str) -> _BreakerA """Reject the call if the breaker is open, else snapshot what its outcome will be judged against.""" if breaker.is_open(): raise RedisCircuitBreakerOpenError(f"Redis circuit breaker is open, skipping {name}") - return _BreakerAdmission(swallowed_before=_swallowed_redis_failures.get(), generation=breaker.generation) + return _BreakerAdmission(swallowed_before=len(_swallowed_redis_failures.get()), generation=breaker.generation) + + +def _take_swallowed_failures(admission: _BreakerAdmission) -> tuple[bool, ...]: + """Return the is_timeout flag of every failure this call swallowed, and drop them from the context.""" + all_swallowed: Final = _swallowed_redis_failures.get() + _swallowed_redis_failures.set(all_swallowed[: admission.swallowed_before]) + return all_swallowed[admission.swallowed_before :] def _exit_circuit_breaker(breaker: RedisCircuitBreaker, admission: _BreakerAdmission) -> None: - """Record success only when nothing failed while the call ran and the breaker has not opened since. + """Report the call's outcome to the breaker generation it was admitted under. Several Redis methods catch their own connection errors and return a default, so a - method that returned is not on its own proof of a healthy Redis. + method that returned is not on its own proof of a healthy Redis. A call admitted + before the breaker opened reports nothing: its failures would refresh the open + timer or knock out the recovery probe, and its success would close it early. """ + swallowed: Final = _take_swallowed_failures(admission) if admission.generation != breaker.generation: return - if _swallowed_redis_failures.get() == admission.swallowed_before: + if not swallowed: breaker.record_success() + return + for is_timeout in swallowed: + breaker.record_failure(is_timeout=is_timeout) def _fail_circuit_breaker(breaker: RedisCircuitBreaker, admission: _BreakerAdmission, exc: BaseException) -> None: - if admission.generation != breaker.generation or not _is_redis_health_failure(exc): + swallowed: Final = _take_swallowed_failures(admission) + if admission.generation != breaker.generation: return - breaker.record_failure(is_timeout=_is_redis_timeout_failure(exc)) + for is_timeout in swallowed: + breaker.record_failure(is_timeout=is_timeout) + if _is_redis_health_failure(exc): + breaker.record_failure(is_timeout=_is_redis_timeout_failure(exc)) async def _run_under_circuit_breaker( @@ -1056,7 +1074,7 @@ class RedisCache(BaseCache): str(e), value, ) - _record_swallowed_redis_failure(self._circuit_breaker, e) + _record_swallowed_redis_failure(e) async def _pipeline_helper( self, @@ -1143,7 +1161,7 @@ class RedisCache(BaseCache): str(e), cache_value, ) - _record_swallowed_redis_failure(self._circuit_breaker, e) + _record_swallowed_redis_failure(e) async def _set_cache_sadd_helper( self, @@ -1228,7 +1246,7 @@ class RedisCache(BaseCache): str(e), value, ) - _record_swallowed_redis_failure(self._circuit_breaker, e) + _record_swallowed_redis_failure(e) @_redis_circuit_breaker_guard async def batch_cache_write(self, key, value, **kwargs): @@ -1386,7 +1404,7 @@ class RedisCache(BaseCache): except Exception as e: # NON blocking - notify users Redis is throwing an exception verbose_logger.error("litellm.caching.caching: get() - Got exception from REDIS: %s", e) - _record_swallowed_redis_failure(self._circuit_breaker, e) + _record_swallowed_redis_failure(e) def _run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]: """ @@ -1423,9 +1441,9 @@ class RedisCache(BaseCache): key_value_dict = {} _key_list: Final = [key for key in key_list if key is not None] start_time: Final = time.time() + admission: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache") try: - admission: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache") _keys: Final = [self.check_and_fix_namespace(key=cache_key or "") for cache_key in _key_list] results: Final = self._run_redis_mget_operation(keys=_keys) _exit_circuit_breaker(self._circuit_breaker, admission) @@ -1452,8 +1470,6 @@ class RedisCache(BaseCache): decoded_results[k] = v return decoded_results - except RedisCircuitBreakerOpenError: - return key_value_dict except Exception as e: failed_at: Final = time.time() self.service_logger_obj.service_failure_hook( @@ -1466,7 +1482,7 @@ class RedisCache(BaseCache): parent_otel_span=parent_otel_span, ) verbose_logger.error("Error occurred in batch get cache - %s", e) - _record_swallowed_redis_failure(self._circuit_breaker, e) + _fail_circuit_breaker(self._circuit_breaker, admission, e) return key_value_dict @_redis_circuit_breaker_guard @@ -1513,7 +1529,7 @@ class RedisCache(BaseCache): ) ) print_verbose(f"litellm.caching.caching: async get() - Got exception from REDIS: {e}") - _record_swallowed_redis_failure(self._circuit_breaker, e) + _record_swallowed_redis_failure(e) @_redis_circuit_breaker_guard async def async_batch_get_cache( @@ -1585,7 +1601,7 @@ class RedisCache(BaseCache): ) ) verbose_logger.error("Error occurred in async batch get cache - %s", e) - _record_swallowed_redis_failure(self._circuit_breaker, e) + _record_swallowed_redis_failure(e) return key_value_dict def sync_ping(self) -> bool: @@ -1837,7 +1853,7 @@ class RedisCache(BaseCache): return ttl except Exception as e: verbose_logger.debug("Redis TTL Error: %s", e) - _record_swallowed_redis_failure(self._circuit_breaker, e) + _record_swallowed_redis_failure(e) return None @_redis_circuit_breaker_guard diff --git a/litellm/litellm_core_utils/logging_worker.py b/litellm/litellm_core_utils/logging_worker.py index dfaace1bb10..de3999d63ea 100644 --- a/litellm/litellm_core_utils/logging_worker.py +++ b/litellm/litellm_core_utils/logging_worker.py @@ -55,8 +55,8 @@ class LoggingWorker: self.max_queue_size = max_queue_size self.concurrency = concurrency self.error_traceback_interval = error_traceback_interval - self._last_error_traceback_at: float | None = None - self._errors_since_traceback: int = 0 + self._last_error_traceback_at: dict[type[BaseException], float] = {} + self._errors_since_traceback: dict[type[BaseException], int] = {} self._queue: asyncio.Queue[LoggingTask] | None = None self._worker_task: asyncio.Task | None = None self._running_tasks: set[asyncio.Task] = set() @@ -178,19 +178,21 @@ class LoggingWorker: sem.release() def _log_task_error(self, error: Exception) -> None: - """One traceback per interval: a stalled backend fails every in-flight task at once.""" + """One traceback per error type per interval: a stalled backend fails every in-flight task at once.""" now: Final = time.monotonic() - last_traceback_at: Final = self._last_error_traceback_at + error_type: Final = type(error) + last_traceback_at: Final = self._last_error_traceback_at.get(error_type) if last_traceback_at is not None and now - last_traceback_at < self.error_traceback_interval: - self._errors_since_traceback += 1 + self._errors_since_traceback[error_type] = self._errors_since_traceback.get(error_type, 0) + 1 return verbose_logger.exception( - "LoggingWorker error (%d more suppressed since the last traceback): %r", - self._errors_since_traceback, + "LoggingWorker error (%d more %s suppressed since the last traceback): %r", + self._errors_since_traceback.get(error_type, 0), + error_type.__name__, error, ) - self._last_error_traceback_at = now - self._errors_since_traceback = 0 + self._last_error_traceback_at[error_type] = now + self._errors_since_traceback[error_type] = 0 async def _worker_loop(self) -> None: """Main worker loop that gets tasks and schedules them to run concurrently.""" diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index 3e21362b3f0..b2462b297fc 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -653,3 +653,23 @@ def test_open_breaker_is_a_quiet_cache_miss_on_the_sync_read_path(dual_cache_wit noisy = [r for r in caplog.records if r.levelno > logging.DEBUG] assert noisy == [], f"an open breaker must be silent per call, got {[r.getMessage() for r in noisy]}" + + +def test_open_breaker_does_not_leave_sync_batch_reservations_behind(dual_cache_with_open_breaker, caplog): + """A batch read skipped by the breaker must not hold its keys for the batch expiry window. + + The sync read reserves keys before dialing Redis so concurrent callers do not all hit it. + When the breaker rejects the read, those reservations have to be released, otherwise the + first read after Redis recovers is still throttled for up to default_redis_batch_cache_expiry. + """ + caplog.set_level(logging.DEBUG, logger="LiteLLM") + dual_cache_with_open_breaker.redis_cache.redis_client.mget.side_effect = AssertionError( + "an open breaker must not touch Redis" + ) + + assert dual_cache_with_open_breaker.batch_get_cache(keys=["lit7468", "lit7460"]) == [None, None] + + assert "lit7468" not in dual_cache_with_open_breaker.last_redis_batch_access_time + assert "lit7460" not in dual_cache_with_open_breaker.last_redis_batch_access_time + noisy = [r for r in caplog.records if r.levelno > logging.DEBUG] + assert noisy == [], f"an open breaker must be silent per call, got {[r.getMessage() for r in noisy]}" diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index 1a884d44db1..3bc3aa1e047 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -516,14 +516,20 @@ async def test_circuit_breaker_opens_when_method_swallows_redis_failure(call_met await call_method(cache) -def test_circuit_breaker_open_keeps_sync_batch_get_cache_as_a_miss(sync_batch_redis_cache): - """An open breaker must preserve the sync batch read's dictionary fallback.""" +def test_sync_batch_get_cache_swallowed_failures_open_the_breaker_and_then_fast_fail(sync_batch_redis_cache): + """The sync batch read hides its Redis error behind an empty dict, but the breaker must still + count it, and once open the read has to raise the typed error like its async twin so DualCache + can release the batch reservations it took before the call. + """ + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD for _ in range(REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD): assert sync_batch_redis_cache.batch_get_cache(key_list=["lit6729"]) == {} - assert sync_batch_redis_cache.batch_get_cache(key_list=["lit6729"]) == {} + with pytest.raises(RedisCircuitBreakerOpenError): + sync_batch_redis_cache.batch_get_cache(key_list=["lit6729"]) + assert sync_batch_redis_cache.redis_client.mget.call_count == REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD def test_batch_get_counts_raises_where_batch_get_cache_reports_a_miss(sync_batch_redis_cache): @@ -647,6 +653,7 @@ def test_sync_batch_get_cache_survives_a_service_callback_that_raises( from concurrent.futures import ThreadPoolExecutor import litellm + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD cache, service_logger = sync_batch_cache_with_service_logger @@ -661,7 +668,8 @@ def test_sync_batch_get_cache_survives_a_service_callback_that_raises( with ThreadPoolExecutor(max_workers=1) as pool: assert pool.submit(cache.batch_get_cache, key_list=["lit6729"]).result() == {} - assert cache.batch_get_cache(key_list=["lit6729"]) == {} + with pytest.raises(RedisCircuitBreakerOpenError): + cache.batch_get_cache(key_list=["lit6729"]) def test_call_stack_info_skips_breaker_guard_frames(): @@ -801,7 +809,7 @@ async def test_concurrent_success_is_not_cancelled_by_another_calls_failure(): # starts would leave its snapshot correct and prove nothing. async def swallows_a_failure(): await asyncio.sleep(0.02) - _record_swallowed_redis_failure(breaker, RedisConnectionError("redis unreachable")) + _record_swallowed_redis_failure(RedisConnectionError("redis unreachable")) async def succeeds_while_the_other_fails(): await asyncio.sleep(0.05) @@ -1097,6 +1105,53 @@ async def test_success_admitted_before_the_breaker_opened_cannot_close_it(): await _run_under_circuit_breaker(breaker, "op", slow_success) +@pytest.mark.asyncio +async def test_swallowed_failure_admitted_before_the_breaker_opened_cannot_delay_recovery(): + """A stale in-flight call that swallows its Redis error must not restart the open timer. + + Guarded methods that return a default instead of raising still report their failure to + the breaker on exit. If that report landed against a newer breaker generation, every + slow call that was already dialing Redis when the breaker opened would push recovery + back by its own socket timeout, and knock out the HALF_OPEN probe if it landed then. + """ + from redis.exceptions import ConnectionError as RedisConnectionError + + from litellm.caching.redis_cache import ( + RedisCircuitBreaker, + _record_swallowed_redis_failure, + _run_under_circuit_breaker, + ) + + breaker = RedisCircuitBreaker(failure_threshold=2, recovery_timeout=0.05) + release_stale_call = asyncio.Event() + + async def stale_call_that_swallows_its_failure(): + await release_stale_call.wait() + _record_swallowed_redis_failure(RedisConnectionError("refused")) + return {} + + async def refused(): + raise RedisConnectionError("refused") + + async def recovered(): + return "ok" + + in_flight = asyncio.create_task(_run_under_circuit_breaker(breaker, "stale", stale_call_that_swallows_its_failure)) + await asyncio.sleep(0) + for _ in range(breaker.failure_threshold): + with pytest.raises(RedisConnectionError): + await _run_under_circuit_breaker(breaker, "op", refused) + assert breaker.is_open() is True + + await asyncio.sleep(0.04) + release_stale_call.set() + assert await in_flight == {} + await asyncio.sleep(0.03) + + assert await _run_under_circuit_breaker(breaker, "probe", recovered) == "ok" + assert breaker.is_open() is False + + @pytest.mark.asyncio async def test_open_breaker_raises_its_own_exception_type(): """Callers with an optional cache need to tell the expected fast-fail apart from a real error.""" diff --git a/tests/test_litellm/litellm_core_utils/test_logging_worker.py b/tests/test_litellm/litellm_core_utils/test_logging_worker.py index f93a2e0b08a..66d49b22c6b 100644 --- a/tests/test_litellm/litellm_core_utils/test_logging_worker.py +++ b/tests/test_litellm/litellm_core_utils/test_logging_worker.py @@ -568,5 +568,33 @@ class TestLoggingWorker: messages = [r.getMessage() for r in caplog.records if "LoggingWorker error" in r.getMessage()] assert len(messages) == 2, messages - assert "(0 more suppressed" in messages[0] - assert "(4 more suppressed" in messages[1] + assert "(0 more TimeoutError suppressed" in messages[0] + assert "(4 more TimeoutError suppressed" in messages[1] + + @pytest.mark.asyncio + async def test_traceback_throttle_is_per_error_type(self, caplog): + """A timeout burst from one stalled backend must not hide the first traceback of a different + failure, otherwise a misconfigured callback stays invisible for the whole interval. + """ + caplog.set_level(logging.DEBUG, logger="LiteLLM") + worker = LoggingWorker(timeout=0.05, max_queue_size=200, concurrency=100, error_traceback_interval=60.0) + worker.start() + + async def stalled_callback(): + await asyncio.sleep(10) + + async def misconfigured_callback(): + raise KeyError("missing api key") + + for _ in range(20): + worker.enqueue(stalled_callback()) + await asyncio.sleep(0.2) + for _ in range(3): + worker.enqueue(misconfigured_callback()) + await asyncio.sleep(0.2) + await worker.stop() + + messages = [r.getMessage() for r in caplog.records if "LoggingWorker error" in r.getMessage()] + assert len(messages) == 2, messages + assert "TimeoutError" in messages[0] + assert "KeyError" in messages[1] and "missing api key" in messages[1] From 368de5df688ef6d21ed47302905aa30e99ee84f5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:30:16 -0700 Subject: [PATCH 089/126] fix(proxy): log prompt-shaped model strings on pass-through failures under the placeholder model --- litellm/constants.py | 1 + .../spend_tracking/spend_tracking_utils.py | 13 +++++-- .../test_spend_tracking_utils.py | 37 ++++++++++++++----- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 33a46616c4e..bc3a82cb655 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -2000,6 +2000,7 @@ NON_INFERENCE_CALL_TYPES: Final[frozenset[str]] = frozenset( ) UNKNOWN_MODEL_SPEND_LOG_MODEL: Final[str] = "unknown-model" +MAX_SPEND_LOG_MODEL_NAME_LENGTH: Final[int] = 256 # PTU reservation rollup writes rows to LiteLLM_DailyTeamSpend with this # sentinel api_key so PTU flat cost stays distinguishable from real per-request diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index ebe5b530e60..d2f44be6004 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -16,6 +16,7 @@ from litellm.constants import ( LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL, @@ -333,6 +334,10 @@ def _sl_attribution_fallback( return standard_logging_payload.get(field) or "" +def _looks_like_model_name(model: str) -> bool: + return len(model) <= MAX_SPEND_LOG_MODEL_NAME_LENGTH and not any(char.isspace() for char in model) + + def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogsPayload: if kwargs is None: kwargs = {} @@ -435,11 +440,13 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs or None ) raw_model: Final = cast(str, kwargs.get("model") or "") + resolved_model: Final = ( + standard_logging_payload.get("model") if standard_logging_payload is not None else None + ) or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) model_name: Final = ( UNKNOWN_MODEL_SPEND_LOG_MODEL - if rejected_as_unknown_model - else (standard_logging_payload.get("model") if standard_logging_payload is not None else None) - or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) + if rejected_as_unknown_model or not _looks_like_model_name(resolved_model) + else resolved_model ) litellm_call_id: Final = cast( str | None, diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 7a20b9ab026..c225b35992f 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -1,8 +1,8 @@ import asyncio import datetime import json -from datetime import timezone from collections.abc import Mapping +from datetime import timezone from typing import Any, Final, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -15,12 +15,13 @@ from litellm.constants import ( LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.spend_tracking.spend_tracking_utils import ( @@ -41,7 +42,6 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( get_logging_payload, get_spend_logs_id, ) -from litellm.proxy._types import SpendLogsPayload from litellm.proxy.utils import hash_token from litellm.types.utils import ( StandardLoggingHiddenParams, @@ -927,21 +927,40 @@ def test_safe_dumps_complex_metadata_like_object(): _RAW_MODEL_WITH_PROMPT: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes" +_BEDROCK_INFERENCE_PROFILE_ARN: Final = ( + "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-sonnet-4-5" +) +_OVERLONG_MODEL: Final = "m" * (MAX_SPEND_LOG_MODEL_NAME_LENGTH + 1) + + @pytest.mark.parametrize( - ("rejection", "expected_model"), + ("requested_model", "failure", "expected_model"), [ ( + _RAW_MODEL_WITH_PROMPT, ProxyModelNotFoundError(route="acompletion", model_name=_RAW_MODEL_WITH_PROMPT), UNKNOWN_MODEL_SPEND_LOG_MODEL, ), - (ValueError("provider timed out"), _RAW_MODEL_WITH_PROMPT), + ( + _RAW_MODEL_WITH_PROMPT, + ValueError("Upstream passthrough request failed with status 404"), + UNKNOWN_MODEL_SPEND_LOG_MODEL, + ), + (_OVERLONG_MODEL, ValueError("provider timed out"), UNKNOWN_MODEL_SPEND_LOG_MODEL), + ( + "gpt-5.2", + ProxyModelNotFoundError(route="acompletion", model_name="gpt-5.2"), + UNKNOWN_MODEL_SPEND_LOG_MODEL, + ), + ("gpt-5.2", ValueError("provider timed out"), "gpt-5.2"), + (_BEDROCK_INFERENCE_PROFILE_ARN, ValueError("provider timed out"), _BEDROCK_INFERENCE_PROFILE_ARN), ], ) -def test_get_logging_payload_replaces_model_only_when_router_rejected_it_as_unknown( - rejection: Exception, expected_model: str +def test_get_logging_payload_replaces_rejected_or_prompt_shaped_models_with_the_placeholder( + requested_model: str, failure: Exception, expected_model: str ): kwargs: Final = { - "model": _RAW_MODEL_WITH_PROMPT, + "model": requested_model, "messages": [{"role": "user", "content": "hi"}], "call_type": "acompletion", "litellm_params": {"metadata": {"user_api_key": "sk-test", "status": "failure"}}, @@ -949,7 +968,7 @@ def test_get_logging_payload_replaces_model_only_when_router_rejected_it_as_unkn payload: Final = get_logging_payload( kwargs=kwargs, - response_obj=rejection, + response_obj=failure, start_time=datetime.datetime.now(timezone.utc), end_time=datetime.datetime.now(timezone.utc), ) From 098bfe0bbc842f43caa15b6b2fc14e2a6106c301 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:36:32 -0700 Subject: [PATCH 090/126] fix(proxy): only remap prompt-shaped model strings on failed spend rows --- .../spend_tracking/spend_tracking_utils.py | 5 ++- .../test_spend_tracking_utils.py | 32 +++++++++++++------ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index d2f44be6004..5690e43ab2b 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -443,9 +443,12 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs resolved_model: Final = ( standard_logging_payload.get("model") if standard_logging_payload is not None else None ) or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) + failed_with_prompt_shaped_model: Final = _get_status_for_spend_log( + metadata=metadata + ) == "failure" and not _looks_like_model_name(resolved_model) model_name: Final = ( UNKNOWN_MODEL_SPEND_LOG_MODEL - if rejected_as_unknown_model or not _looks_like_model_name(resolved_model) + if rejected_as_unknown_model or failed_with_prompt_shaped_model else resolved_model ) litellm_call_id: Final = cast( diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index c225b35992f..11cf7ac530e 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -141,9 +141,7 @@ def test_batch_lifecycle_rows_derive_the_same_session_from_the_batch_id(): from litellm.proxy.spend_tracking.spend_tracking_utils import _get_batch_trace_session_id create_session: Final = _get_batch_trace_session_id(call_type="acreate_batch", request_id="batch-uid-1") - cost_session: Final = _get_batch_trace_session_id( - call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost" - ) + cost_session: Final = _get_batch_trace_session_id(call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost") assert create_session == cost_session == "batch-uid-1" @@ -976,6 +974,24 @@ def test_get_logging_payload_replaces_rejected_or_prompt_shaped_models_with_the_ assert payload["model"] == expected_model +def test_get_logging_payload_keeps_a_whitespace_model_name_on_success(): + kwargs: Final = { + "model": _RAW_MODEL_WITH_PROMPT, + "messages": [{"role": "user", "content": "hi"}], + "call_type": "acompletion", + "litellm_params": {"metadata": {"user_api_key": "sk-test"}}, + } + + payload: Final = get_logging_payload( + kwargs=kwargs, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert payload["model"] == _RAW_MODEL_WITH_PROMPT + + @patch("litellm.proxy.proxy_server.master_key", None) @patch("litellm.proxy.proxy_server.general_settings", {}) def test_get_logging_payload_api_key_preserved_when_standard_logging_payload_is_none(): @@ -4306,7 +4322,7 @@ ANTHROPIC_MESSAGES_SSE_CHUNKS: Final = ( 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n', 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},' '"usage":{"output_tokens":4}}\n\n', - "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", + 'event: message_stop\ndata: {"type":"message_stop"}\n\n', ) @@ -4344,9 +4360,7 @@ def test_spend_log_request_id_is_the_message_id_a_non_streaming_messages_caller_ """ logging_obj = _anthropic_messages_logging_obj(stream=False) - logged_response = logging_obj._handle_anthropic_messages_response_logging( - result=ANTHROPIC_MESSAGES_RESPONSE - ) + logged_response = logging_obj._handle_anthropic_messages_response_logging(result=ANTHROPIC_MESSAGES_RESPONSE) assert logged_response.id == "msg_01Lit6806NonStreaming" assert ( @@ -4422,9 +4436,7 @@ def test_spend_log_request_id_still_falls_back_to_litellm_call_id_without_a_prov end_time=datetime.datetime.now(timezone.utc), logging_obj=logging_obj, ) - assert logging_obj.model_call_details["complete_streaming_response"].id == ( - "6806cafe-0000-4000-8000-000000000001" - ) + assert logging_obj.model_call_details["complete_streaming_response"].id == ("6806cafe-0000-4000-8000-000000000001") def test_spend_log_request_id_for_chat_completions_is_untouched(): From a93753e73bfc67de48a8b068d590b72b6d7425c2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:43:27 -0700 Subject: [PATCH 091/126] fix(proxy): keep the model on failed rows that resolved to a configured model group --- .../spend_tracking/spend_tracking_utils.py | 8 +++-- .../test_spend_tracking_utils.py | 32 +++++++++++++++---- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 5690e43ab2b..f8cd73938b4 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -443,9 +443,11 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs resolved_model: Final = ( standard_logging_payload.get("model") if standard_logging_payload is not None else None ) or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) - failed_with_prompt_shaped_model: Final = _get_status_for_spend_log( - metadata=metadata - ) == "failure" and not _looks_like_model_name(resolved_model) + failed_with_prompt_shaped_model: Final = ( + _get_status_for_spend_log(metadata=metadata) == "failure" + and not _model_group + and not _looks_like_model_name(resolved_model) + ) model_name: Final = ( UNKNOWN_MODEL_SPEND_LOG_MODEL if rejected_as_unknown_model or failed_with_prompt_shaped_model diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 11cf7ac530e..03c34e9e198 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -141,7 +141,9 @@ def test_batch_lifecycle_rows_derive_the_same_session_from_the_batch_id(): from litellm.proxy.spend_tracking.spend_tracking_utils import _get_batch_trace_session_id create_session: Final = _get_batch_trace_session_id(call_type="acreate_batch", request_id="batch-uid-1") - cost_session: Final = _get_batch_trace_session_id(call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost") + cost_session: Final = _get_batch_trace_session_id( + call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost" + ) assert create_session == cost_session == "batch-uid-1" @@ -974,17 +976,29 @@ def test_get_logging_payload_replaces_rejected_or_prompt_shaped_models_with_the_ assert payload["model"] == expected_model -def test_get_logging_payload_keeps_a_whitespace_model_name_on_success(): +@pytest.mark.parametrize( + ("metadata", "response_obj"), + [ + ({"user_api_key": "sk-test"}, litellm.ModelResponse(id="chatcmpl-test", choices=[])), + ( + {"user_api_key": "sk-test", "model_group": "team alias", "status": "failure"}, + ValueError("provider timed out"), + ), + ], +) +def test_get_logging_payload_keeps_a_whitespace_model_name_on_success_or_a_routed_failure( + metadata: dict[str, str], response_obj: litellm.ModelResponse | Exception +): kwargs: Final = { "model": _RAW_MODEL_WITH_PROMPT, "messages": [{"role": "user", "content": "hi"}], "call_type": "acompletion", - "litellm_params": {"metadata": {"user_api_key": "sk-test"}}, + "litellm_params": {"metadata": metadata}, } payload: Final = get_logging_payload( kwargs=kwargs, - response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + response_obj=response_obj, start_time=datetime.datetime.now(timezone.utc), end_time=datetime.datetime.now(timezone.utc), ) @@ -4322,7 +4336,7 @@ ANTHROPIC_MESSAGES_SSE_CHUNKS: Final = ( 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n', 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},' '"usage":{"output_tokens":4}}\n\n', - 'event: message_stop\ndata: {"type":"message_stop"}\n\n', + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", ) @@ -4360,7 +4374,9 @@ def test_spend_log_request_id_is_the_message_id_a_non_streaming_messages_caller_ """ logging_obj = _anthropic_messages_logging_obj(stream=False) - logged_response = logging_obj._handle_anthropic_messages_response_logging(result=ANTHROPIC_MESSAGES_RESPONSE) + logged_response = logging_obj._handle_anthropic_messages_response_logging( + result=ANTHROPIC_MESSAGES_RESPONSE + ) assert logged_response.id == "msg_01Lit6806NonStreaming" assert ( @@ -4436,7 +4452,9 @@ def test_spend_log_request_id_still_falls_back_to_litellm_call_id_without_a_prov end_time=datetime.datetime.now(timezone.utc), logging_obj=logging_obj, ) - assert logging_obj.model_call_details["complete_streaming_response"].id == ("6806cafe-0000-4000-8000-000000000001") + assert logging_obj.model_call_details["complete_streaming_response"].id == ( + "6806cafe-0000-4000-8000-000000000001" + ) def test_spend_log_request_id_for_chat_completions_is_untouched(): From 032a9ebc4b370eb6f53ca00ce0755b069ddc1331 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 15:46:01 -0700 Subject: [PATCH 092/126] chore(github): disable blank issues so filers must use a template (#40629) --- .github/ISSUE_TEMPLATE/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index cbf380bac01..fc86c229c81 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,4 +1,4 @@ -blank_issues_enabled: true +blank_issues_enabled: false contact_links: - name: Schedule Demo url: https://enterprise.litellm.ai/demo From fbd923190e04583e56184c6ff38e1b2e41ad6c36 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 10 Sep 2026 22:53:46 +0000 Subject: [PATCH 093/126] fix(caching): let the outer breaker guard judge failures a nested guarded call swallowed batch_cache_write flushes through async_set_cache_pipeline, both guarded. The inner guard consumed the swallowed pipeline failure and the outer then recorded a success, so a dead Redis never tripped the breaker on the batch write path. Only the outermost admission now reports, and the sync batch read runs under the same guard. Also covers the sync add_cache and embedding pipeline wrappers, the increment pipeline, sadd, and the stale raise path in tests. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/redis_cache.py | 36 ++++++-- tests/test_litellm/caching/test_caching.py | 33 +++++-- tests/test_litellm/caching/test_dual_cache.py | 5 ++ .../test_litellm/caching/test_redis_cache.py | 90 +++++++++++++++++-- 4 files changed, 141 insertions(+), 23 deletions(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 3ecf805ed0e..2c1ca75212c 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -288,6 +288,7 @@ _RedisCallResult = TypeVar("_RedisCallResult") _swallowed_redis_failures: Final[ContextVar[tuple[bool, ...]]] = ContextVar( "litellm_swallowed_redis_failures", default=() ) +_breaker_depth: Final[ContextVar[int]] = ContextVar("litellm_redis_breaker_depth", default=0) def _opaque_kwarg_key(value: object) -> str: @@ -425,13 +426,20 @@ def _record_swallowed_redis_failure(exc: BaseException) -> None: class _BreakerAdmission: swallowed_before: int generation: int + nested: bool def _enter_circuit_breaker(breaker: RedisCircuitBreaker, name: str) -> _BreakerAdmission: """Reject the call if the breaker is open, else snapshot what its outcome will be judged against.""" if breaker.is_open(): raise RedisCircuitBreakerOpenError(f"Redis circuit breaker is open, skipping {name}") - return _BreakerAdmission(swallowed_before=len(_swallowed_redis_failures.get()), generation=breaker.generation) + depth: Final = _breaker_depth.get() + _breaker_depth.set(depth + 1) + return _BreakerAdmission( + swallowed_before=len(_swallowed_redis_failures.get()), + generation=breaker.generation, + nested=depth > 0, + ) def _take_swallowed_failures(admission: _BreakerAdmission) -> tuple[bool, ...]: @@ -448,7 +456,14 @@ def _exit_circuit_breaker(breaker: RedisCircuitBreaker, admission: _BreakerAdmis method that returned is not on its own proof of a healthy Redis. A call admitted before the breaker opened reports nothing: its failures would refresh the open timer or knock out the recovery probe, and its success would close it early. + + A guarded method calling another guarded method is one Redis interaction, so only + the outermost admission reports. The inner one leaves its swallowed failures in the + context for the outer to judge, otherwise the outer would read a clean context and + reset the streak the inner just fed. """ + if admission.nested: + return swallowed: Final = _take_swallowed_failures(admission) if admission.generation != breaker.generation: return @@ -459,7 +474,13 @@ def _exit_circuit_breaker(breaker: RedisCircuitBreaker, admission: _BreakerAdmis breaker.record_failure(is_timeout=is_timeout) +def _leave_circuit_breaker() -> None: + _breaker_depth.set(_breaker_depth.get() - 1) + + def _fail_circuit_breaker(breaker: RedisCircuitBreaker, admission: _BreakerAdmission, exc: BaseException) -> None: + if admission.nested: + return swallowed: Final = _take_swallowed_failures(admission) if admission.generation != breaker.generation: return @@ -485,6 +506,8 @@ async def _run_under_circuit_breaker( except Exception as e: _fail_circuit_breaker(breaker, admission, e) raise + finally: + _leave_circuit_breaker() _exit_circuit_breaker(breaker, admission) return result @@ -501,6 +524,8 @@ def _run_under_circuit_breaker_sync( except Exception as e: _fail_circuit_breaker(breaker, admission, e) raise + finally: + _leave_circuit_breaker() _exit_circuit_breaker(breaker, admission) return result @@ -1441,12 +1466,12 @@ class RedisCache(BaseCache): key_value_dict = {} _key_list: Final = [key for key in key_list if key is not None] start_time: Final = time.time() - admission: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache") try: _keys: Final = [self.check_and_fix_namespace(key=cache_key or "") for cache_key in _key_list] - results: Final = self._run_redis_mget_operation(keys=_keys) - _exit_circuit_breaker(self._circuit_breaker, admission) + results: Final = _run_under_circuit_breaker_sync( + self._circuit_breaker, "batch_get_cache", lambda: self._run_redis_mget_operation(keys=_keys) + ) end_time: Final = time.time() _duration: Final = end_time - start_time self.service_logger_obj.service_success_hook( @@ -1470,6 +1495,8 @@ class RedisCache(BaseCache): decoded_results[k] = v return decoded_results + except RedisCircuitBreakerOpenError: + raise except Exception as e: failed_at: Final = time.time() self.service_logger_obj.service_failure_hook( @@ -1482,7 +1509,6 @@ class RedisCache(BaseCache): parent_otel_span=parent_otel_span, ) verbose_logger.error("Error occurred in batch get cache - %s", e) - _fail_circuit_breaker(self._circuit_breaker, admission, e) return key_value_dict @_redis_circuit_breaker_guard diff --git a/tests/test_litellm/caching/test_caching.py b/tests/test_litellm/caching/test_caching.py index 5d326a9ece0..d135d2892c5 100644 --- a/tests/test_litellm/caching/test_caching.py +++ b/tests/test_litellm/caching/test_caching.py @@ -255,26 +255,41 @@ def test_exact_cache_key_includes_anthropic_messages_params(anthropic_param): @pytest.mark.asyncio -async def test_async_add_cache_treats_an_open_breaker_as_a_quiet_skip(caplog): - """The top-level write wrapper logs a full ERROR traceback for any failure. An open Redis - breaker fails every write instantly, so under load that wrapper alone was hundreds of +@pytest.mark.parametrize("write", ["add_cache", "async_add_cache", "async_add_cache_pipeline"]) +async def test_add_cache_wrappers_treat_an_open_breaker_as_a_quiet_skip(caplog, write: str): + """The top-level write wrappers log a full ERROR traceback for any failure. An open Redis + breaker fails every write instantly, so under load those wrappers alone were hundreds of stack formats per second per replica. Unexpected failures must still get the traceback. """ - from unittest.mock import AsyncMock + from unittest.mock import AsyncMock, MagicMock from litellm.caching.redis_cache import RedisCircuitBreakerOpenError + from litellm.types.utils import Embedding, EmbeddingResponse caplog.set_level(logging.DEBUG, logger="LiteLLM") cache = Cache(type=LiteLLMCacheType.LOCAL) - cache.cache.async_set_cache = AsyncMock(side_effect=RedisCircuitBreakerOpenError("open")) + embedding = EmbeddingResponse(model="text-embedding-3-small", data=[Embedding(embedding=[0.1], index=0, object="embedding")]) - await cache.async_add_cache("result", model="gpt-5.4-nano", messages=[{"role": "user", "content": "hi"}]) + async def run_write() -> None: + if write == "add_cache": + cache.add_cache("result", model="gpt-5.4-nano", messages=[{"role": "user", "content": "hi"}]) + elif write == "async_add_cache": + await cache.async_add_cache("result", model="gpt-5.4-nano", messages=[{"role": "user", "content": "hi"}]) + else: + await cache.async_add_cache_pipeline(embedding, model="text-embedding-3-small", input="hi") + + def install_backend(exc: Exception) -> None: + cache.cache.set_cache = MagicMock(side_effect=exc) + cache.cache.async_set_cache = AsyncMock(side_effect=exc) + cache.cache.async_set_cache_pipeline = AsyncMock(side_effect=exc) + + install_backend(RedisCircuitBreakerOpenError("open")) + await run_write() assert [r.levelno for r in caplog.records if "add_cache" in r.getMessage()] == [logging.DEBUG] - cache.cache.async_set_cache.assert_awaited_once() - cache.cache.async_set_cache = AsyncMock(side_effect=OSError("disk full")) - await cache.async_add_cache("result", model="gpt-5.4-nano", messages=[{"role": "user", "content": "hi"}]) + install_backend(OSError("disk full")) + await run_write() errors = [r for r in caplog.records if r.levelno == logging.ERROR] assert len(errors) == 1 and errors[0].exc_info is not None diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index b2462b297fc..4177c215d0b 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -621,6 +621,11 @@ def dual_cache_with_open_breaker(): pytest.param(lambda c: c.async_set_cache("lit7468", "v"), lambda n: None, id="async_set_cache"), pytest.param(lambda c: c.async_set_cache_pipeline([("lit7468", "v")]), lambda n: None, id="async_set_cache_pipeline"), pytest.param(lambda c: c.async_increment_cache("lit7468", 1.0, ttl=60), float, id="async_increment_cache"), + pytest.param( + lambda c: c.async_increment_cache_pipeline([{"key": "lit7468", "increment_value": 1.0, "ttl": 60}]), + lambda n: [float(n)], + id="async_increment_cache_pipeline", + ), ], ) async def test_open_breaker_is_a_quiet_cache_miss(dual_cache_with_open_breaker, call, expected, caplog): diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index 3bc3aa1e047..8c994a4742e 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -494,6 +494,7 @@ def _closed_port() -> int: pytest.param(lambda c: c.async_batch_get_cache(["lit4930"]), id="async_batch_get_cache"), pytest.param(lambda c: c.async_set_cache("lit4930", "v"), id="async_set_cache"), pytest.param(lambda c: c.async_get_ttl("lit4930"), id="async_get_ttl"), + pytest.param(lambda c: c.async_set_cache_sadd("lit4930", ["v"], ttl=None), id="async_set_cache_sadd"), ], ) async def test_circuit_breaker_opens_when_method_swallows_redis_failure(call_method): @@ -505,6 +506,7 @@ async def test_circuit_breaker_opens_when_method_swallows_redis_failure(call_met breaker could never open. An unreachable Redis then stayed in the pool and every request kept paying the full socket timeout on it. """ + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD cache = await asyncio.to_thread(RedisCache, host="127.0.0.1", port=_closed_port(), socket_timeout=0.5) @@ -512,10 +514,73 @@ async def test_circuit_breaker_opens_when_method_swallows_redis_failure(call_met for _ in range(REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD): await call_method(cache) - with pytest.raises(Exception, match="circuit breaker is open"): + with pytest.raises(RedisCircuitBreakerOpenError): await call_method(cache) +@pytest.mark.asyncio +async def test_nested_guarded_flush_failures_still_open_the_breaker(): + """A guarded method that delegates to another guarded method is one Redis call, not two. + + batch_cache_write is guarded and flushes through the guarded async_set_cache_pipeline, + which swallows the pipeline error. If the inner guard consumes that failure, the outer + guard sees a clean run and records a success, so the streak resets on every write and + a dead Redis keeps receiving flushes instead of tripping the breaker. + """ + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError + from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD + + cache = await asyncio.to_thread( + RedisCache, host="127.0.0.1", port=_closed_port(), socket_timeout=0.5, redis_flush_size=1 + ) + + for _ in range(REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD): + await cache.batch_cache_write("lit7468", "v") + + with pytest.raises(RedisCircuitBreakerOpenError): + await cache.batch_cache_write("lit7468", "v") + assert cache._circuit_breaker._failure_count == REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD + + +@pytest.mark.asyncio +async def test_nested_guard_that_raises_counts_one_failure_for_the_outer_call(): + """An inner guarded call that raises through the outer one is a single Redis failure, and a + swallowed inner failure followed by an outer raise is two, so the count follows what Redis + actually refused rather than how many guard frames the error crossed. + """ + from redis.exceptions import ConnectionError as RedisConnectionError + + from litellm.caching.redis_cache import ( + RedisCircuitBreaker, + _record_swallowed_redis_failure, + _run_under_circuit_breaker, + ) + + breaker = RedisCircuitBreaker(failure_threshold=10, recovery_timeout=60) + + async def refused(): + raise RedisConnectionError("refused") + + async def outer_delegating_to_inner(): + return await _run_under_circuit_breaker(breaker, "inner", refused) + + async def outer_swallowing_then_raising(): + async def inner_swallowing(): + _record_swallowed_redis_failure(RedisConnectionError("refused")) + return {} + + await _run_under_circuit_breaker(breaker, "inner", inner_swallowing) + raise RedisConnectionError("refused") + + with pytest.raises(RedisConnectionError): + await _run_under_circuit_breaker(breaker, "outer", outer_delegating_to_inner) + assert breaker._failure_count == 1 + + with pytest.raises(RedisConnectionError): + await _run_under_circuit_breaker(breaker, "outer", outer_swallowing_then_raising) + assert breaker._failure_count == 3 + + def test_sync_batch_get_cache_swallowed_failures_open_the_breaker_and_then_fast_fail(sync_batch_redis_cache): """The sync batch read hides its Redis error behind an empty dict, but the breaker must still count it, and once open the read has to raise the typed error like its async twin so DualCache @@ -1106,12 +1171,13 @@ async def test_success_admitted_before_the_breaker_opened_cannot_close_it(): @pytest.mark.asyncio -async def test_swallowed_failure_admitted_before_the_breaker_opened_cannot_delay_recovery(): - """A stale in-flight call that swallows its Redis error must not restart the open timer. +@pytest.mark.parametrize("stale_call_raises", [False, True], ids=["swallows", "raises"]) +async def test_failure_admitted_before_the_breaker_opened_cannot_delay_recovery(stale_call_raises: bool): + """A stale in-flight call that fails after the breaker opened must not restart the open timer. - Guarded methods that return a default instead of raising still report their failure to - the breaker on exit. If that report landed against a newer breaker generation, every - slow call that was already dialing Redis when the breaker opened would push recovery + Whether the method swallows its Redis error and returns a default or lets it propagate, + the failure is reported on exit. If that report landed against a newer breaker generation, + every slow call that was already dialing Redis when the breaker opened would push recovery back by its own socket timeout, and knock out the HALF_OPEN probe if it landed then. """ from redis.exceptions import ConnectionError as RedisConnectionError @@ -1125,8 +1191,10 @@ async def test_swallowed_failure_admitted_before_the_breaker_opened_cannot_delay breaker = RedisCircuitBreaker(failure_threshold=2, recovery_timeout=0.05) release_stale_call = asyncio.Event() - async def stale_call_that_swallows_its_failure(): + async def stale_call(): await release_stale_call.wait() + if stale_call_raises: + raise RedisConnectionError("refused") _record_swallowed_redis_failure(RedisConnectionError("refused")) return {} @@ -1136,7 +1204,7 @@ async def test_swallowed_failure_admitted_before_the_breaker_opened_cannot_delay async def recovered(): return "ok" - in_flight = asyncio.create_task(_run_under_circuit_breaker(breaker, "stale", stale_call_that_swallows_its_failure)) + in_flight = asyncio.create_task(_run_under_circuit_breaker(breaker, "stale", stale_call)) await asyncio.sleep(0) for _ in range(breaker.failure_threshold): with pytest.raises(RedisConnectionError): @@ -1145,7 +1213,11 @@ async def test_swallowed_failure_admitted_before_the_breaker_opened_cannot_delay await asyncio.sleep(0.04) release_stale_call.set() - assert await in_flight == {} + if stale_call_raises: + with pytest.raises(RedisConnectionError): + await in_flight + else: + assert await in_flight == {} await asyncio.sleep(0.03) assert await _run_under_circuit_breaker(breaker, "probe", recovered) == "ok" From 71d1bfb70a7f9ad2cac9e4e2090f029791fbd2e7 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 15:55:03 -0700 Subject: [PATCH 094/126] feat(ui): link the User, Team and Created By cells on the Virtual Keys page The key detail page already walks out to the user, team and org behind a key, but the Virtual Keys table rendered those same values as dead text, so getting to a team meant copying its alias and searching the Teams page. User, Team and Created By now render through the shared IdentityCell with an href, the same hover-highlight-and-chevron affordance the Key column already uses. Sentinel ids do not get a link, since they have no detail page to open. Rather than repeat that check at every call site, teamDetailHref and userDetailHref now return undefined for "litellm-dashboard" and "default_user_id", the way modelGroupHref already does for model grants, and EntityLink falls back to plain text when it has no href, the way BadgeLink already does. Both sentinels move into src/utils/sentinels.ts instead of staying as string literals scattered across components. Claude-Session: https://claude.ai/code/session_01NfwfQhamRNnSqgXMUjf3h4 --- .../_components/AccessGroupsDetailsPage.tsx | 2 +- .../VirtualKeysPage/VirtualKeysTable.test.tsx | 71 +++++++++++++++++++ .../VirtualKeysPage/keyTableColumns.tsx | 45 ++++++------ .../DefaultProxyAdminTag.tsx | 5 +- .../common_components/LabeledField.tsx | 3 +- .../src/components/shared/EntityLink.test.tsx | 6 ++ .../src/components/shared/EntityLink.tsx | 14 +++- .../src/utils/entityLinks.test.ts | 24 ++++++- ui/litellm-dashboard/src/utils/entityLinks.ts | 7 +- ui/litellm-dashboard/src/utils/sentinels.ts | 3 + 10 files changed, 147 insertions(+), 33 deletions(-) create mode 100644 ui/litellm-dashboard/src/utils/sentinels.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx index 1eeebe4ebba..118a8655d5a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx @@ -54,7 +54,7 @@ function ResourceBadge({ fallback, }: { resource: AccessGroupResource; - href: string; + href?: string; fallback: (id: string) => string; }) { const badge = ( diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index 881b0b93ff9..57786d71c66 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -473,6 +473,77 @@ it("should display 'Default Proxy Admin' for user_id when value is 'default_user }); }); +describe("entity links out of the key rows", () => { + const keyRow = async () => (await screen.findByText("Test Key Alias")).closest("tr") as HTMLElement; + + const enableCreatedByColumn = async (user: ReturnType) => { + await user.click(screen.getByRole("button", { name: "Columns" })); + await user.click(await screen.findByText("Created By")); + await user.keyboard("{Escape}"); + }; + + it("points the User and Team cells at their detail pages", async () => { + renderWithProviders(); + + const row = await keyRow(); + expect(within(row).getByRole("link", { name: "user@example.com" })).toHaveAttribute( + "href", + "/ui/users?user=user-1", + ); + expect(within(row).getByRole("link", { name: "Test Team" })).toHaveAttribute("href", "/ui/teams?team=team-1"); + }); + + it("points the Created By cell at the creator's detail page", async () => { + mockUseKeys.mockReturnValue( + keysResult([ + { + ...mockKey, + created_by: "creator-1", + created_by_user: { user_id: "creator-1", user_email: "creator@example.com", user_alias: "The Creator" }, + }, + ]), + ); + const user = userEvent.setup(); + renderWithProviders(); + await enableCreatedByColumn(user); + + const row = await keyRow(); + expect(within(row).getByRole("link", { name: "The Creator" })).toHaveAttribute("href", "/ui/users?user=creator-1"); + }); + + it("leaves the default_user_id placeholder unlinked even once it resolves to a named user", async () => { + const placeholder = { user_id: "default_user_id", user_email: "admin@example.com", user_alias: "Proxy Admin" }; + mockUseKeys.mockReturnValue( + keysResult([ + { + ...mockKey, + user_id: placeholder.user_id, + user_email: placeholder.user_email, + user: placeholder, + created_by: placeholder.user_id, + created_by_user: placeholder, + }, + ]), + ); + const user = userEvent.setup(); + renderWithProviders(); + await enableCreatedByColumn(user); + + const row = await keyRow(); + expect(within(row).getAllByText("Proxy Admin")).toHaveLength(2); + expect(within(row).queryByRole("link", { name: "Proxy Admin" })).not.toBeInTheDocument(); + }); + + it("leaves the litellm-dashboard session team unlinked", async () => { + mockUseKeys.mockReturnValue(keysResult([{ ...mockKey, team_id: "litellm-dashboard" }])); + renderWithProviders(); + + const row = await keyRow(); + expect(within(row).getByText("litellm-dashboard")).toBeInTheDocument(); + expect(within(row).queryByRole("link", { name: "litellm-dashboard" })).not.toBeInTheDocument(); + }); +}); + it("should render table without crashing when models is null", async () => { mockUseKeys.mockReturnValue(keysResult([{ ...mockKey, models: null as unknown as string[] }])); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx index 0865fe76519..c87bc90fb47 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx @@ -16,6 +16,8 @@ import { StatusBadge, type StatusTone, } from "@/components/shared/table_cells"; +import { teamDetailHref, userDetailHref } from "@/utils/entityLinks"; +import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; import { KeyResponse, Team } from "../key_team_helpers/key_list"; @@ -27,6 +29,8 @@ interface KeyStatus { tooltip?: string; } +const ENTITY_CELL_TITLE_CLASSES = "font-mono text-xs font-normal"; + const SPEND_BUDGET_SORT_FIELDS: DataTableSortField[] = [ { id: "spend", label: "Spend" }, { id: "max_budget", label: "Budget" }, @@ -74,7 +78,7 @@ const UserPopoverCell = ({ width: number; }) => { const displayValue = userAlias || userEmail || userId; - const isDefaultAdmin = userId === "default_user_id"; + const isDefaultAdmin = userId === DEFAULT_PROXY_ADMIN_USER_ID; const popoverContent = (
@@ -95,28 +99,21 @@ const UserPopoverCell = ({
); - if (isDefaultAdmin && !userAlias && !userEmail) { - return ( - - }> - - - {popoverContent} - + const trigger = + isDefaultAdmin && !userAlias && !userEmail ? ( + + ) : ( + ); - } return ( - - } - > - {displayValue || "-"} + }> + {trigger} {popoverContent} @@ -201,12 +198,12 @@ export const getKeyTableColumns = ({ const teamId = info.getValue() as string | null; if (!teamId) return "-"; const team = allTeams.find((t) => t.team_id === teamId); - const displayValue = team?.team_alias || teamId; - const width = info.cell.column.getSize(); return ( - - {displayValue} - + ); }, }, diff --git a/ui/litellm-dashboard/src/components/common_components/DefaultProxyAdminTag.tsx b/ui/litellm-dashboard/src/components/common_components/DefaultProxyAdminTag.tsx index 9ec24bb929b..308f722a8f2 100644 --- a/ui/litellm-dashboard/src/components/common_components/DefaultProxyAdminTag.tsx +++ b/ui/litellm-dashboard/src/components/common_components/DefaultProxyAdminTag.tsx @@ -1,13 +1,12 @@ import { Badge } from "@/components/ui/badge"; - -const DEFAULT_USER_ID = "default_user_id"; +import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; interface DefaultProxyAdminTagProps { userId: string | null | undefined; } export default function DefaultProxyAdminTag({ userId }: DefaultProxyAdminTagProps) { - if (userId === DEFAULT_USER_ID) { + if (userId === DEFAULT_PROXY_ADMIN_USER_ID) { return Default Proxy Admin; } diff --git a/ui/litellm-dashboard/src/components/common_components/LabeledField.tsx b/ui/litellm-dashboard/src/components/common_components/LabeledField.tsx index 9f45b05f306..99046cfa84d 100644 --- a/ui/litellm-dashboard/src/components/common_components/LabeledField.tsx +++ b/ui/litellm-dashboard/src/components/common_components/LabeledField.tsx @@ -2,6 +2,7 @@ import React from "react"; import CopyButton from "@/components/shared/CopyButton"; import { EntityLink } from "@/components/shared/EntityLink"; import { cx } from "@/lib/cva.config"; +import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; import DefaultProxyAdminTag from "./DefaultProxyAdminTag"; interface LabeledFieldProps { @@ -24,7 +25,7 @@ export default function LabeledField({ defaultUserIdCheck = false, }: LabeledFieldProps) { const isEmpty = !value; - const isDefaultUser = defaultUserIdCheck && value === "default_user_id"; + const isDefaultUser = defaultUserIdCheck && value === DEFAULT_PROXY_ADMIN_USER_ID; const displayValue = isEmpty ? "-" : value; const isCopyable = copyable && !isEmpty && !isDefaultUser; const isLink = href != null && !isEmpty && !isDefaultUser; diff --git a/ui/litellm-dashboard/src/components/shared/EntityLink.test.tsx b/ui/litellm-dashboard/src/components/shared/EntityLink.test.tsx index a02f1699c30..3d6fc9a435b 100644 --- a/ui/litellm-dashboard/src/components/shared/EntityLink.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/EntityLink.test.tsx @@ -25,6 +25,12 @@ describe("EntityLink", () => { expect(push).toHaveBeenCalledWith("/ui/users?user=u1"); }); + it("renders the label as plain text when there is no href to point at", () => { + render(default_user_id); + expect(screen.queryByRole("link")).not.toBeInTheDocument(); + expect(screen.getByText("default_user_id")).toBeInTheDocument(); + }); + it("leaves modified clicks to the browser so new-tab shortcuts keep working", async () => { const user = userEvent.setup(); render(alice); diff --git a/ui/litellm-dashboard/src/components/shared/EntityLink.tsx b/ui/litellm-dashboard/src/components/shared/EntityLink.tsx index 4054b929943..4885835a42c 100644 --- a/ui/litellm-dashboard/src/components/shared/EntityLink.tsx +++ b/ui/litellm-dashboard/src/components/shared/EntityLink.tsx @@ -19,12 +19,24 @@ export function useEntityLinkClick(href: string): (e: React.MouseEvent) => void } interface EntityLinkProps { - href: string; + href?: string; className?: string; children: React.ReactNode; } export function EntityLink({ href, className, children }: EntityLinkProps) { + if (!href) { + return {children}; + } + + return ( + + {children} + + ); +} + +function LinkedEntity({ href, className, children }: EntityLinkProps & { href: string }) { const handleClick = useEntityLinkClick(href); return ( diff --git a/ui/litellm-dashboard/src/utils/entityLinks.test.ts b/ui/litellm-dashboard/src/utils/entityLinks.test.ts index 47161a903ed..231413b213a 100644 --- a/ui/litellm-dashboard/src/utils/entityLinks.test.ts +++ b/ui/litellm-dashboard/src/utils/entityLinks.test.ts @@ -2,7 +2,29 @@ import { describe, expect, it, vi } from "vitest"; vi.mock("@/components/networking", () => ({ serverRootPath: "" })); -import { modelGroupHref } from "./entityLinks"; +import { modelGroupHref, teamDetailHref, userDetailHref } from "./entityLinks"; + +describe("userDetailHref", () => { + it("targets the users page filtered to the encoded user id", () => { + expect(userDetailHref("user-1")).toMatch(/\/users\?user=user-1$/); + expect(userDetailHref("a b/c")).toMatch(/\?user=a%20b%2Fc$/); + }); + + it("returns no href for the proxy admin placeholder, which has no user page", () => { + expect(userDetailHref("default_user_id")).toBeUndefined(); + }); +}); + +describe("teamDetailHref", () => { + it("targets the teams page filtered to the encoded team id", () => { + expect(teamDetailHref("team-1")).toMatch(/\/teams\?team=team-1$/); + expect(teamDetailHref("a b/c")).toMatch(/\?team=a%20b%2Fc$/); + }); + + it("returns no href for the Admin UI session team, which has no team page", () => { + expect(teamDetailHref("litellm-dashboard")).toBeUndefined(); + }); +}); describe("modelGroupHref", () => { it("targets the models page filtered to the encoded model group", () => { diff --git a/ui/litellm-dashboard/src/utils/entityLinks.ts b/ui/litellm-dashboard/src/utils/entityLinks.ts index b8c70bdda48..33f2aa34976 100644 --- a/ui/litellm-dashboard/src/utils/entityLinks.ts +++ b/ui/litellm-dashboard/src/utils/entityLinks.ts @@ -1,3 +1,4 @@ +import { DEFAULT_PROXY_ADMIN_USER_ID, UI_TEAM_ID } from "@/utils/sentinels"; import { uiHref } from "@/utils/uiHref"; const MODEL_GRANT_SENTINELS: ReadonlySet = new Set([ @@ -6,7 +7,8 @@ const MODEL_GRANT_SENTINELS: ReadonlySet = new Set([ "no-default-models", ]); -export function teamDetailHref(teamId: string): string { +export function teamDetailHref(teamId: string): string | undefined { + if (teamId === UI_TEAM_ID) return undefined; return `${uiHref("teams")}?team=${encodeURIComponent(teamId)}`; } @@ -14,7 +16,8 @@ export function keyDetailHref(keyToken: string): string { return `${uiHref("api-keys")}?key=${encodeURIComponent(keyToken)}`; } -export function userDetailHref(userId: string): string { +export function userDetailHref(userId: string): string | undefined { + if (userId === DEFAULT_PROXY_ADMIN_USER_ID) return undefined; return `${uiHref("users")}?user=${encodeURIComponent(userId)}`; } diff --git a/ui/litellm-dashboard/src/utils/sentinels.ts b/ui/litellm-dashboard/src/utils/sentinels.ts new file mode 100644 index 00000000000..da6d09dfdd7 --- /dev/null +++ b/ui/litellm-dashboard/src/utils/sentinels.ts @@ -0,0 +1,3 @@ +export const DEFAULT_PROXY_ADMIN_USER_ID = "default_user_id"; + +export const UI_TEAM_ID = "litellm-dashboard"; From 311441ced19b1963f8ad22b758a8ae6171105a74 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 10 Sep 2026 16:08:57 -0700 Subject: [PATCH 095/126] bump: litellm-proxy-extras 0.4.95 -> 0.4.96 --- litellm-proxy-extras/pyproject.toml | 4 ++-- pyproject.toml | 2 +- uv.lock | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 91b4e4a7ba1..7d4c78088f1 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.95" +version = "0.4.96" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.95" +version = "0.4.96" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/pyproject.toml b/pyproject.toml index 04f2f3fd1dd..d33d693f794 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,7 @@ proxy = [ "azure-identity>=1.25.2,<2.0", "azure-storage-blob>=12.28.0,<13.0", "mcp>=1.28.1,<2.0", - "litellm-proxy-extras==0.4.95", + "litellm-proxy-extras==0.4.96", "litellm-enterprise==0.1.66", "RestrictedPython>=8.5,<9.0", "rich>=13.9.4,<14.0", diff --git a/uv.lock b/uv.lock index 0fe787645a2..cedc505acdf 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-09-06T00:40:30.433549Z" +exclude-newer = "2026-09-07T23:09:03.362777Z" exclude-newer-span = "P3D" [manifest] @@ -4772,7 +4772,7 @@ source = { editable = "enterprise" } [[package]] name = "litellm-proxy-extras" -version = "0.4.95" +version = "0.4.96" source = { editable = "litellm-proxy-extras" } [[package]] From 3b93da414ead03604bb9dcba0b24be585db28639 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:18:00 -0700 Subject: [PATCH 096/126] feat(proxy): let the in-container pgbouncer follow rotating RDS IAM and Azure Entra tokens (#40623) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- gateway/launch.py | 20 +- helm/litellm/templates/_helpers.tpl | 5 +- helm/litellm/tests/connection_pool_tests.yaml | 28 +- helm/litellm/values.yaml | 6 +- litellm/proxy/db/db_url_settings.py | 5 + litellm/proxy/db/pgbouncer.py | 280 +++++++++++++--- litellm/proxy/proxy_cli.py | 12 +- litellm/proxy/utils.py | 6 +- terraform/litellm/aws/README.md | 11 +- terraform/litellm/aws/ecs.tf | 4 +- .../aws/tests/connection_pool.tftest.hcl | 12 +- terraform/litellm/aws/variables.tf | 7 +- tests/test_gateway/test_launch.py | 75 ++++- tests/test_litellm/proxy/db/test_pgbouncer.py | 314 ++++++++++++++++-- tests/test_litellm/proxy/test_proxy_utils.py | 49 ++- 15 files changed, 703 insertions(+), 131 deletions(-) diff --git a/gateway/launch.py b/gateway/launch.py index 6b28fafdcf6..d67432caee1 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -4,8 +4,9 @@ is fine for a plain Postgres URL but not for the pooler: PgBouncer must be started exactly once per pod, before the workers fork, and the workers must be handed the loopback URL it listens on. A pre-existing ``DATABASE_URL`` wins in -``DatabaseURLSettings.apply_to_env`` under password auth, so setting it here is -enough for every worker to pick the pooled URL up unchanged. +``DatabaseURLSettings.apply_to_env`` under password auth, and one marked pooled +wins under token auth too, so exporting it here is enough for every worker to +pick the pooled URL up unchanged. Run with: python -m gateway.launch --workers 4 --host 0.0.0.0 --port 4000 @@ -19,7 +20,12 @@ from typing import Final from uvicorn.main import main as uvicorn_main from litellm.proxy.db.db_url_settings import DatabaseURLSettings -from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, start_in_container_pgbouncer +from litellm.proxy.db.pgbouncer import ( + PgBouncerError, + PgBouncerSettings, + export_pooled_database_url, + start_in_container_pgbouncer, +) GATEWAY_APP: Final = "gateway.main:app" KEEPALIVE_FLAG: Final = "--timeout-keep-alive" @@ -41,15 +47,15 @@ def pool_database_url( """Start the in-container PgBouncer and return its loopback URL, or None when ``pgbouncer.enabled`` is off. The upstream URL is whatever ``apply_to_env`` assembled from the discrete - ``DATABASE_*`` vars (or an operator-pinned ``DATABASE_URL``). Token auth is - rejected by the pooler itself, since it holds one password for its lifetime. + ``DATABASE_*`` vars (or an operator-pinned ``DATABASE_URL``). Under token + auth the pooler mints and renews the upstream token itself. """ if not pgbouncer.enabled: return None upstream_url: Final = environ.get("DATABASE_URL") if upstream_url is None: return PgBouncerError("LITELLM_PGBOUNCER_ENABLED is set but no DATABASE_URL could be assembled") - return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth_enabled=settings.token_auth() is not None) + return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth()) def _serve(argv: Sequence[str]) -> None: @@ -63,7 +69,7 @@ def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) - if isinstance(pooled_url, PgBouncerError): sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}") if pooled_url is not None: - os.environ["DATABASE_URL"] = pooled_url + export_pooled_database_url(pooled_url) serve(uvicorn_argv(argv, os.environ)) diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 4ad3cfe0484..dc47bc0b0a1 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -361,12 +361,9 @@ harmless no-op for the Job and authoritative for the app pods. {{- end -}} {{/* -In-container PgBouncer env for the gateway container. Fails at render time under IAM or Entra auth: the pooler holds one static password for the life of the pod. +In-container PgBouncer env for the gateway container. Under IAM or Entra auth the pooler mints and renews the database token itself. */}} {{- define "litellm.connectionPoolEnv" -}} -{{- if or .Values.database.writer.useIAMAuth .Values.database.writer.useAzureEntraAuth }} -{{- fail "database.connectionPool.enabled cannot be combined with database.writer.useIAMAuth or database.writer.useAzureEntraAuth: the in-container pgbouncer holds a static database password and cannot follow a rotating token. Disable the pool or use a static database password" }} -{{- end }} {{- with .Values.database.connectionPool -}} - name: LITELLM_PGBOUNCER_ENABLED value: "true" diff --git a/helm/litellm/tests/connection_pool_tests.yaml b/helm/litellm/tests/connection_pool_tests.yaml index 31be7575f3c..6dd9c274a8c 100644 --- a/helm/litellm/tests/connection_pool_tests.yaml +++ b/helm/litellm/tests/connection_pool_tests.yaml @@ -82,23 +82,39 @@ tests: name: LITELLM_PGBOUNCER_ENABLED any: true - - it: pool with IAM auth fails at render time + - it: pool with IAM auth renders both the pool and the token auth flag template: gateway/deployment.yaml set: database.connectionPool.enabled: true database.writer.useIAMAuth: true asserts: - - failedTemplate: - errorMessage: "database.connectionPool.enabled cannot be combined with database.writer.useIAMAuth or database.writer.useAzureEntraAuth: the in-container pgbouncer holds a static database password and cannot follow a rotating token. Disable the pool or use a static database password" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + value: "true" + - contains: + path: spec.template.spec.containers[0].env + content: + name: IAM_TOKEN_DB_AUTH + value: "true" - - it: pool with Entra auth fails at render time + - it: pool with Entra auth renders both the pool and the token auth flag template: gateway/deployment.yaml set: database.connectionPool.enabled: true database.writer.useAzureEntraAuth: true asserts: - - failedTemplate: - errorMessage: "database.connectionPool.enabled cannot be combined with database.writer.useIAMAuth or database.writer.useAzureEntraAuth: the in-container pgbouncer holds a static database password and cannot follow a rotating token. Disable the pool or use a static database password" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + value: "true" + - contains: + path: spec.template.spec.containers[0].env + content: + name: AZURE_POSTGRESQL_AUTH + value: "true" - it: IAM auth without the pool still renders template: gateway/deployment.yaml diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 45ab5229f38..677e21a5a51 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -235,9 +235,9 @@ database: # network hop. The chart emits LITELLM_PGBOUNCER_ENABLED / # LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS / LITELLM_PGBOUNCER_MAX_CLIENT_CONN on # the gateway container only: the backend runs a single worker and the - # migrations Job must keep a direct connection. The pool holds a static - # password, so it cannot be combined with `database.writer.useIAMAuth` or - # `useAzureEntraAuth` (rendering fails). Starting profile for + # migrations Job must keep a direct connection. With + # `database.writer.useIAMAuth` or `useAzureEntraAuth` the pool mints and + # renews the database token itself, so the workers never see it. Starting profile for # `gateway.numWorkers: 4` is maxDbConnections: 20, so a database with a # 5000-connection ceiling fits roughly 200 gateway replicas. connectionPool: diff --git a/litellm/proxy/db/db_url_settings.py b/litellm/proxy/db/db_url_settings.py index d1e4b3e92b8..e93bc96da69 100644 --- a/litellm/proxy/db/db_url_settings.py +++ b/litellm/proxy/db/db_url_settings.py @@ -52,6 +52,7 @@ from typing import Annotated, Final, Protocol, TypeAlias, cast from pydantic import AliasChoices, BeforeValidator, Field from pydantic_settings import BaseSettings, SettingsConfigDict +from litellm.proxy.db.pgbouncer import database_url_is_pooled from litellm.proxy.db.token_auth import ( AZURE_POSTGRESQL_AUTH_ENV_VAR, DEFAULT_POSTGRES_PORT, @@ -358,8 +359,12 @@ class DatabaseURLSettings(BaseSettings): Raises ``RuntimeError`` (naming the offending vars) when token auth is enabled but a required field is missing — the proxy cannot recover from this and a clear startup error beats a Prisma connect failure. + A ``DATABASE_URL`` the supervisor pointed at the in-container PgBouncer + is kept even under token auth: the pooler renews the token upstream. """ auth: Final = self.token_auth() + if auth is not None and database_url_is_pooled(): + return None if auth is not None: missing: Final = tuple( env diff --git a/litellm/proxy/db/pgbouncer.py b/litellm/proxy/db/pgbouncer.py index 7792867600c..eac80f7a712 100644 --- a/litellm/proxy/db/pgbouncer.py +++ b/litellm/proxy/db/pgbouncer.py @@ -18,19 +18,28 @@ Migrations and the schema diff run in the supervisor before the pooler is started, so they always go straight to Postgres. ``DATABASE_URL_READ_REPLICA`` is left untouched. -The pooler holds the database password from startup, so it cannot be combined -with ``IAM_TOKEN_DB_AUTH`` or ``AZURE_POSTGRESQL_AUTH``: those rotate the -password inside every worker on their own schedule, and PgBouncer would keep -authenticating upstream with the expired token. +The workers never hold the upstream credential: they log in to PgBouncer as +``litellm_pgbouncer`` with a random password made at startup, and PgBouncer +takes the database user's password from its auth file. Under +``IAM_TOKEN_DB_AUTH`` or ``AZURE_POSTGRESQL_AUTH`` that password is a +short-lived token, so the supervisor mints a new one before it expires, +rewrites the auth file and asks PgBouncer to reload; only new upstream +connections authenticate, so live ones are unaffected. The pooled +``DATABASE_URL`` then carries a static password, and the workers must not run +their own token refresh against it: ``LITELLM_PGBOUNCER_POOLED_DATABASE_URL`` +tells them so, while a read replica keeps refreshing its own token. """ from __future__ import annotations import atexit +import functools import os import re +import secrets import shlex import shutil +import signal import socket import subprocess import tempfile @@ -39,6 +48,7 @@ import time import urllib.parse from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path from types import MappingProxyType from typing import Final @@ -47,10 +57,18 @@ from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict from litellm._logging import verbose_proxy_logger -from litellm.proxy.db.token_auth import AZURE_POSTGRESQL_AUTH_ENV_VAR, IAM_TOKEN_DB_AUTH_ENV_VAR +from litellm.proxy.db.token_auth import ( + DatabaseTokenAuth, + IAMEndpoint, + mint_database_token, + parse_database_token_expiration, + parse_iam_endpoint_from_url, +) PGBOUNCER_ENV_PREFIX: Final = "LITELLM_PGBOUNCER_" +PGBOUNCER_POOLED_ENV_VAR: Final = "LITELLM_PGBOUNCER_POOLED_DATABASE_URL" PGBOUNCER_LISTEN_ADDR: Final = "127.0.0.1" +PGBOUNCER_POOL_USER: Final = "litellm_pgbouncer" PGBOUNCER_INI_NAME: Final = "pgbouncer.ini" PGBOUNCER_USERLIST_NAME: Final = "userlist.txt" PGBOUNCER_CA_NAME: Final = "server-ca.pem" @@ -59,13 +77,11 @@ PGBOUNCER_READY_TIMEOUT_SECONDS: Final = 15.0 PGBOUNCER_STOP_GRACE_SECONDS: Final = 10.0 PGBOUNCER_UNPRIVILEGED_USER: Final = "nobody" PGBOUNCER_MIN_VERSION: Final = (1, 19) +PGBOUNCER_MAX_PASSWORD_BYTES: Final = 2048 PGBOUNCER_VERSION_PATTERN: Final = re.compile(r"PgBouncer (\d+)\.(\d+)") -PGBOUNCER_LIST_DELIMITER_PATTERN: Final = re.compile(r"[,\s]") -PGBOUNCER_TOKEN_AUTH_CONFLICT: Final = ( - f"the in-container pgbouncer cannot be combined with {IAM_TOKEN_DB_AUTH_ENV_VAR} or " - f"{AZURE_POSTGRESQL_AUTH_ENV_VAR}: each worker rotates the database password on its own schedule and the pooler " - "would keep using the expired token upstream. Disable the pooler or use a static database password" -) +PGBOUNCER_TOKEN_REFRESH_BUFFER_SECONDS: Final = 180.0 +PGBOUNCER_TOKEN_FALLBACK_REFRESH_SECONDS: Final = 600.0 +PGBOUNCER_TOKEN_RETRY_SECONDS: Final = 30.0 # Prisma's client-side TLS params describe the hop to Postgres, which becomes # PgBouncer's server side. They move into ``server_tls_*`` and must not stay on @@ -97,10 +113,18 @@ class PgBouncerSettings(BaseSettings): @dataclass(frozen=True, slots=True) class PgBouncerPlan: ini: str - userlist: str pooled_url: str + upstream_user: str + upstream_password: str | None + pool_password: str ca_source: str | None = None + def userlist(self, upstream_password: str) -> str: + return "".join( + f"{_userlist_quote(user)} {_userlist_quote(password)}\n" + for user, password in ((self.upstream_user, upstream_password), (PGBOUNCER_POOL_USER, self.pool_password)) + ) + @dataclass(frozen=True, slots=True) class PgBouncerError: @@ -173,7 +197,9 @@ def plan_pgbouncer( Params describing Prisma's own pool (``connection_limit``, ``pool_timeout``, ...) stay on the pooled URL; the TLS params and ``options`` describe the hop - to Postgres and move into the PgBouncer config. ``run_as_user`` is the + to Postgres and move into the PgBouncer config. The upstream password is + left out of the config on purpose: PgBouncer then takes it from the auth + file, which can be rewritten while it runs. ``run_as_user`` is the unprivileged user PgBouncer drops to when the proxy runs as root, which PgBouncer itself refuses to do. """ @@ -184,14 +210,12 @@ def plan_pgbouncer( dbname: Final = urllib.parse.unquote(parsed.path.lstrip("/")) username: Final = urllib.parse.unquote(parsed.username or "") password: Final = None if parsed.password is None else urllib.parse.unquote(parsed.password) - if not parsed.hostname or not username or password is None or not dbname: + if not parsed.hostname or not username or not dbname: + return PgBouncerError("DATABASE_URL must carry a host, user and database name for the in-container PgBouncer") + if username == PGBOUNCER_POOL_USER: return PgBouncerError( - "DATABASE_URL must carry a host, user, password and database name for the in-container PgBouncer" - ) - if PGBOUNCER_LIST_DELIMITER_PATTERN.search(username): - return PgBouncerError( - f"the database user {username!r} cannot be named in PgBouncer's stats_users list: " - "PgBouncer splits list settings on commas and whitespace and has no quoting for them" + f"the database user cannot be named {PGBOUNCER_POOL_USER!r}: that is the user the workers log in to the " + "in-container PgBouncer as, and PgBouncer keeps one password per user" ) if "sslidentity" in params: return PgBouncerError("client certificates (sslidentity) are not supported with the in-container PgBouncer") @@ -212,7 +236,6 @@ def plan_pgbouncer( f"port={parsed.port or 5432}", f"dbname={_single_quoted(dbname)}", f"user={_single_quoted(username)}", - f"password={_single_quoted(password)}", *((f"connect_query={_single_quoted(connect_query)}",) if connect_query else ()), ) ) @@ -227,7 +250,7 @@ def plan_pgbouncer( f"unix_socket_dir = {runtime_dir}", f"auth_file = {runtime_dir / PGBOUNCER_USERLIST_NAME}", "auth_type = scram-sha-256", - f"stats_users = {username}", + f"stats_users = {PGBOUNCER_POOL_USER}", "pool_mode = transaction", f"max_client_conn = {settings.max_client_conn}", f"default_pool_size = {settings.max_db_connections}", @@ -238,41 +261,175 @@ def plan_pgbouncer( "", ) ) - userlist: Final = f"{_userlist_quote(username)} {_userlist_quote(password)}\n" pooled_query: Final = urllib.parse.urlencode( (*((key, value) for key, value in params.items() if key not in POOLED_URL_DROPPED_KEYS), ("pgbouncer", "true")) ) - credentials: Final = f"{urllib.parse.quote(username, safe='')}:{urllib.parse.quote(password, safe='')}" + pool_password: Final = secrets.token_urlsafe(32) pooled_url: Final = urllib.parse.urlunsplit( - parsed._replace(netloc=f"{credentials}@{PGBOUNCER_LISTEN_ADDR}:{settings.port}", query=pooled_query) + parsed._replace( + netloc=f"{PGBOUNCER_POOL_USER}:{pool_password}@{PGBOUNCER_LISTEN_ADDR}:{settings.port}", query=pooled_query + ) + ) + return PgBouncerPlan( + ini=ini, + pooled_url=pooled_url, + upstream_user=username, + upstream_password=password, + pool_password=pool_password, + ca_source=params.get("sslcert") or None, ) - return PgBouncerPlan(ini=ini, userlist=userlist, pooled_url=pooled_url, ca_source=params.get("sslcert") or None) -def write_pgbouncer_files(plan: PgBouncerPlan, runtime_dir: Path, run_as_user: str | None) -> Path | PgBouncerError: - """Write the ini, userlist (both hold the password, so mode 0600) and CA copy, and return the ini path. +def _write_private(path: Path, content: str, run_as_user: str | None) -> None: + with open(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "w", encoding="utf-8") as handle: + handle.write(content) + if run_as_user is not None: + shutil.chown(path, user=run_as_user) + + +def write_pgbouncer_ini(plan: PgBouncerPlan, runtime_dir: Path, run_as_user: str | None) -> Path | PgBouncerError: + """Write the ini (mode 0600) and the CA copy, and return the ini path. ``run_as_user`` is the user PgBouncer drops to when started as root; it has to own the files it re-reads on reload and the socket directory. """ ini_path: Final = runtime_dir / PGBOUNCER_INI_NAME - userlist_path: Final = runtime_dir / PGBOUNCER_USERLIST_NAME ca_path: Final = runtime_dir / PGBOUNCER_CA_NAME if plan.ca_source is not None: try: shutil.copyfile(plan.ca_source, ca_path) except OSError as error: return PgBouncerError(f"cannot read the CA bundle {plan.ca_source!r} named by sslcert: {error}") - for path, content in ((userlist_path, plan.userlist), (ini_path, plan.ini)): - path.touch(mode=0o600) - path.write_text(content, encoding="utf-8") + _write_private(ini_path, plan.ini, run_as_user) if run_as_user is not None: runtime_dir.chmod(0o700) - for path in (runtime_dir, ini_path, userlist_path, *((ca_path,) if plan.ca_source is not None else ())): + for path in (runtime_dir, *((ca_path,) if plan.ca_source is not None else ())): shutil.chown(path, user=run_as_user) return ini_path +def write_userlist(userlist: str, runtime_dir: Path, run_as_user: str | None) -> Path: + """Replace the auth file in one step, so a PgBouncer starting or reloading meanwhile reads the old or the new one whole.""" + userlist_path: Final = runtime_dir / PGBOUNCER_USERLIST_NAME + staged_path: Final = runtime_dir / f".{PGBOUNCER_USERLIST_NAME}.next" + _write_private(staged_path, userlist, run_as_user) + os.replace(staged_path, userlist_path) + return userlist_path + + +def export_pooled_database_url(pooled_url: str) -> None: + os.environ["DATABASE_URL"] = pooled_url + os.environ[PGBOUNCER_POOLED_ENV_VAR] = "true" + + +def database_url_is_pooled(environ: Mapping[str, str] = os.environ) -> bool: + return environ.get(PGBOUNCER_POOLED_ENV_VAR) == "true" + + +@dataclass(frozen=True, slots=True) +class PgBouncerTokenSource: + auth: DatabaseTokenAuth + endpoint: IAMEndpoint + + def mint(self) -> str: + """The token as Postgres expects it: ``mint_database_token`` returns it percent-encoded for a URL.""" + return urllib.parse.unquote(mint_database_token(self.auth, self.endpoint)) + + def expires_at(self, token: str) -> datetime | None: + return parse_database_token_expiration(self.auth, token) + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc).replace(tzinfo=None) + + +class PgBouncerTokenRefresher: + """Keeps the token in PgBouncer's auth file current from a daemon thread. + + ``install`` gets each fresh token and is expected to rewrite the auth file + and reload PgBouncer. The next refresh is due ``buffer_seconds`` before the + token expires, or ``fallback_seconds`` later when the expiry cannot be read. + A refresh that fails leaves the previous auth file in place and is retried + after ``retry_seconds``: the old token stays good until it expires, so a + transient credential-provider error costs nothing unless it persists. + """ + + def __init__( + self, + source: PgBouncerTokenSource, + install: Callable[[str], None], + *, + buffer_seconds: float = PGBOUNCER_TOKEN_REFRESH_BUFFER_SECONDS, + fallback_seconds: float = PGBOUNCER_TOKEN_FALLBACK_REFRESH_SECONDS, + retry_seconds: float = PGBOUNCER_TOKEN_RETRY_SECONDS, + now: Callable[[], datetime] = _utcnow, + ) -> None: + self._source: Final = source + self._install: Final = install + self._buffer_seconds: Final = buffer_seconds + self._fallback_seconds: Final = fallback_seconds + self._retry_seconds: Final = retry_seconds + self._now: Final = now + self._stopping: Final = threading.Event() + self._delay: float = 0.0 + self._thread: threading.Thread | None = None + + def refresh(self) -> float | PgBouncerError: + label: Final = self._source.auth.label + try: + token: Final = self._source.mint() + except Exception as mint_error: + return PgBouncerError(f"could not mint a {label} for the in-container pgbouncer: {mint_error!r}") + if len(token.encode()) >= PGBOUNCER_MAX_PASSWORD_BYTES: + return PgBouncerError( + f"the {label} is {len(token.encode())} bytes long, but PgBouncer's auth file holds passwords of at " + f"most {PGBOUNCER_MAX_PASSWORD_BYTES - 1} bytes" + ) + try: + self._install(token) + except OSError as install_error: + return PgBouncerError(f"could not install the {label} into the pgbouncer auth file: {install_error}") + expires_at: Final = self._source.expires_at(token) + if expires_at is None: + return self._fallback_seconds + return max(self._retry_seconds, (expires_at - self._now()).total_seconds() - self._buffer_seconds) + + def start(self) -> PgBouncerError | None: + primed: Final = self.refresh() + if isinstance(primed, PgBouncerError): + return primed + self._delay = primed + self._thread = threading.Thread(target=self._run, daemon=True, name="litellm-pgbouncer-token-refresh") + self._thread.start() + return None + + def _run(self) -> None: + while not self._stopping.wait(self._delay): + self._delay = self._refresh_and_report() + + def _refresh_and_report(self) -> float: + outcome: Final = self.refresh() + if isinstance(outcome, PgBouncerError): + verbose_proxy_logger.error( + "In-container pgbouncer keeps its current %s (%s); retrying in %.0fs.", + self._source.auth.label, + outcome.reason, + self._retry_seconds, + ) + return self._retry_seconds + verbose_proxy_logger.info( + "In-container pgbouncer picked up a fresh %s; the next one is due in %.0fs.", + self._source.auth.label, + outcome, + ) + return outcome + + def stop(self) -> None: + self._stopping.set() + if self._thread is not None: + self._thread.join() + + def _port_open(port: int) -> bool: try: with socket.create_connection((PGBOUNCER_LISTEN_ADDR, port), timeout=0.5): @@ -453,6 +610,11 @@ class PgBouncerProcess: ) threading.Thread(target=self._restart_after_delay, daemon=True, name="litellm-pgbouncer-supervisor").start() + def reload(self) -> None: + with self._lock: + if self._process is not None: + self._process.send_signal(signal.SIGHUP) + def stop(self) -> None: with self._lock: self._stopping.set() @@ -461,6 +623,39 @@ class PgBouncerProcess: _end(process) +def install_pgbouncer_token( + plan: PgBouncerPlan, runtime_dir: Path, run_as_user: str | None, pooler: PgBouncerProcess, token: str +) -> None: + write_userlist(plan.userlist(token), runtime_dir, run_as_user) + pooler.reload() + + +def _install_upstream_password( + plan: PgBouncerPlan, + runtime_dir: Path, + run_as_user: str | None, + pooler: PgBouncerProcess, + token_auth: DatabaseTokenAuth | None, + upstream_url: str, +) -> PgBouncerTokenRefresher | None | PgBouncerError: + if token_auth is None: + if plan.upstream_password is None: + return PgBouncerError( + "DATABASE_URL carries no password and neither IAM_TOKEN_DB_AUTH nor AZURE_POSTGRESQL_AUTH is on, " + "so the in-container PgBouncer has nothing to authenticate to Postgres with" + ) + write_userlist(plan.userlist(plan.upstream_password), runtime_dir, run_as_user) + return None + refresher: Final = PgBouncerTokenRefresher( + PgBouncerTokenSource(auth=token_auth, endpoint=parse_iam_endpoint_from_url(upstream_url)), + functools.partial(install_pgbouncer_token, plan, runtime_dir, run_as_user, pooler), + ) + failed: Final = refresher.start() + if failed is not None: + return failed + return refresher + + def _only_in_this_process(action: Callable[[], None]) -> Callable[[], None]: """An exit hook that does nothing in a forked child, which inherits the parent's ``atexit`` table.""" owner_pid: Final = os.getpid() @@ -475,7 +670,7 @@ def _only_in_this_process(action: Callable[[], None]) -> Callable[[], None]: def start_in_container_pgbouncer( settings: PgBouncerSettings, upstream_url: str, - token_auth_enabled: bool = False, + token_auth: DatabaseTokenAuth | None = None, register_exit_hook: Callable[[Callable[[], None]], object] = atexit.register, ) -> str | PgBouncerError: """Start the pooler for ``upstream_url`` and return the loopback URL the workers must use. @@ -484,10 +679,9 @@ def start_in_container_pgbouncer( once the worker manager has returned, and only by the process that started it (gunicorn forks its workers, so they carry the hooks too). PgBouncer refuses to run as root, so a root proxy (the default image) has it drop to - ``nobody``. + ``nobody``. With ``token_auth`` the password on ``upstream_url`` is ignored: + the pooler mints its own tokens and renews them for as long as it runs. """ - if token_auth_enabled: - return PgBouncerError(PGBOUNCER_TOKEN_AUTH_CONFLICT) version: Final = pgbouncer_version(settings.binary) if isinstance(version, PgBouncerError): return version @@ -503,7 +697,7 @@ def start_in_container_pgbouncer( plan: Final = plan_pgbouncer(upstream_url, settings, runtime_dir, run_as_user) if isinstance(plan, PgBouncerError): return plan - ini_path: Final = write_pgbouncer_files(plan, runtime_dir, run_as_user) + ini_path: Final = write_pgbouncer_ini(plan, runtime_dir, run_as_user) if isinstance(ini_path, PgBouncerError): return ini_path pooler: Final = PgBouncerProcess( @@ -511,15 +705,23 @@ def start_in_container_pgbouncer( port=settings.port, socket_path=unix_socket_path(runtime_dir, settings.port), ) + refresher: Final = _install_upstream_password(plan, runtime_dir, run_as_user, pooler, token_auth, upstream_url) + if isinstance(refresher, PgBouncerError): + return refresher failed: Final = pooler.start() if failed is not None: + if refresher is not None: + refresher.stop() return failed register_exit_hook(_only_in_this_process(pooler.stop)) + if refresher is not None: + register_exit_hook(_only_in_this_process(refresher.stop)) verbose_proxy_logger.info( - "In-container pgbouncer (pid %s) listening on %s:%s; capping this pod at %s upstream database connections.", + "In-container pgbouncer (pid %s) listening on %s:%s; capping this pod at %s upstream database connections%s.", pooler.pid, PGBOUNCER_LISTEN_ADDR, settings.port, settings.max_db_connections, + "" if token_auth is None else f" and renewing its {token_auth.label} before each one expires", ) return plan.pooled_url diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 16d76ff0415..c2d60cd5488 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -18,7 +18,12 @@ from pydantic import BaseModel, ConfigDict import litellm from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY -from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, start_in_container_pgbouncer +from litellm.proxy.db.pgbouncer import ( + PgBouncerError, + PgBouncerSettings, + export_pooled_database_url, + start_in_container_pgbouncer, +) from litellm.proxy.db.query_engine_reaper import start_query_engine_reaper if TYPE_CHECKING: @@ -1109,6 +1114,7 @@ def run_server( from litellm.proxy.db.token_auth import ( AZURE_POSTGRESQL_AUTH_ENV_VAR, IAM_TOKEN_DB_AUTH_ENV_VAR, + resolve_database_token_auth, token_auth_flag_enabled, ) @@ -1382,7 +1388,7 @@ def run_server( upstream_database_url: Final = os.getenv("DATABASE_URL") if pgbouncer_settings.enabled and upstream_database_url is not None: pooled_database_url: Final = start_in_container_pgbouncer( - pgbouncer_settings, upstream_database_url, token_auth_enabled=wants_rds_iam or wants_azure_entra + pgbouncer_settings, upstream_database_url, token_auth=resolve_database_token_auth() ) if isinstance(pooled_database_url, PgBouncerError): print( @@ -1392,7 +1398,7 @@ def run_server( flush=True, ) sys.exit(1) - os.environ["DATABASE_URL"] = pooled_database_url + export_pooled_database_url(pooled_database_url) if port == 4000 and ProxyInitializationHelpers._is_port_in_use(port): port = random.randint(1024, 49152) if prometheus_metrics_port == port: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 2cc58ca3110..253494b02f4 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -131,6 +131,7 @@ from litellm.proxy.db.health_check_latest import ( fetch_latest_health_checks_for_models, ) from litellm.proxy.db.log_db_metrics import log_db_metrics +from litellm.proxy.db.pgbouncer import database_url_is_pooled from litellm.proxy.db.prisma_client import ( PrismaWrapper, parse_iam_endpoint_from_url, @@ -4007,6 +4008,7 @@ class PrismaClient: verbose_proxy_logger.error("Please run 'prisma generate' to generate the Prisma client.") raise Exception("Unable to find Prisma binaries. Please run 'prisma generate' first.") token_auth: Final = self.token_auth + writer_token_auth: Final = None if database_url_is_pooled() else token_auth # When read-replica routing is on, tag log lines with [writer]/[reader] # so the two wrappers' interleaved token refresh logs can be told apart. # Single-DB deployments get an empty prefix (logs unchanged). @@ -4015,13 +4017,13 @@ class PrismaClient: if http_client is not None: writer_wrapper = PrismaWrapper( original_prisma=Prisma(http=http_client), - token_auth=token_auth, + token_auth=writer_token_auth, log_prefix=writer_log_prefix, ) else: writer_wrapper = PrismaWrapper( original_prisma=Prisma(), - token_auth=token_auth, + token_auth=writer_token_auth, log_prefix=writer_log_prefix, ) diff --git a/terraform/litellm/aws/README.md b/terraform/litellm/aws/README.md index 986428151e0..5ca45944c54 100644 --- a/terraform/litellm/aws/README.md +++ b/terraform/litellm/aws/README.md @@ -273,18 +273,17 @@ connections the pooler accepts. The module sets and the migration task keep the direct connection. ```hcl -create_database = false -database_url = "postgresql://litellm:@db.internal:5432/litellm" gateway_num_workers = 4 gateway_connection_pool_enabled = true gateway_pool_max_db_connections = 20 gateway_pool_max_client_conn = 1000 ``` -The pool needs a static database password, so it is only valid with an -existing database via `database_url`. The module-created Aurora authenticates -with rotating IAM tokens (see [Aurora + IAM auth](#aurora--iam-auth)), which -the pooler cannot follow, and `terraform plan` rejects that combination. +The pool works with the module-created Aurora as well as an existing database +via `database_url`. Against Aurora it authenticates with the same rotating IAM +tokens the workers used to (see [Aurora + IAM auth](#aurora--iam-auth)): the +pooler mints a token from the task role, renews it before it expires and hands +the workers a loopback URL with a static password instead The componentized `gateway_image` starts through `python -m gateway.launch`, which reads these variables, starts the pooler once per task and hands the diff --git a/terraform/litellm/aws/ecs.tf b/terraform/litellm/aws/ecs.tf index 149842c14ff..7176921e8fa 100644 --- a/terraform/litellm/aws/ecs.tf +++ b/terraform/litellm/aws/ecs.tf @@ -306,8 +306,8 @@ resource "aws_ecs_task_definition" "gateway" { } precondition { - condition = !var.gateway_connection_pool_enabled || local.byo_database - error_message = "gateway_connection_pool_enabled requires an existing database via database_url with create_database = false: the module-created Aurora authenticates with IAM tokens, which the in-container pgbouncer cannot follow because it holds a static database password." + condition = !var.gateway_connection_pool_enabled || local.database_enabled + error_message = "gateway_connection_pool_enabled needs a database: set create_database = true or pass database_url." } } diff --git a/terraform/litellm/aws/tests/connection_pool.tftest.hcl b/terraform/litellm/aws/tests/connection_pool.tftest.hcl index 38aa7051134..ab0a1cfc076 100644 --- a/terraform/litellm/aws/tests/connection_pool.tftest.hcl +++ b/terraform/litellm/aws/tests/connection_pool.tftest.hcl @@ -88,16 +88,20 @@ run "gateway_starts_through_the_pool_aware_launcher" { } } -run "pool_with_module_created_iam_aurora_fails_at_plan" { +run "pool_with_module_created_iam_aurora_plans_with_both_the_pool_and_iam_auth" { command = plan variables { gateway_connection_pool_enabled = true } - expect_failures = [ - aws_ecs_task_definition.gateway, - ] + assert { + condition = alltrue([ + length(local.gateway_pool_env) == 3, + contains(local.managed_db_env, { name = "IAM_TOKEN_DB_AUTH", value = "true" }), + ]) + error_message = "With the module-created Aurora the gateway must get the pool env alongside IAM token auth." + } } run "pool_without_any_database_fails_at_plan" { diff --git a/terraform/litellm/aws/variables.tf b/terraform/litellm/aws/variables.tf index ec8363dbbaf..f57bb1ab7f6 100644 --- a/terraform/litellm/aws/variables.tf +++ b/terraform/litellm/aws/variables.tf @@ -208,10 +208,9 @@ variable "gateway_connection_pool_enabled" { Postgres, so a task's footprint against the database connection ceiling is workers x connection_limit and grows with every task. Sets LITELLM_PGBOUNCER_ENABLED / LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS / - LITELLM_PGBOUNCER_MAX_CLIENT_CONN on the gateway container only. Requires - an existing database via `database_url`: the module-created Aurora - authenticates with IAM tokens, which the pooler cannot follow because it - holds one static password for the life of the task. + LITELLM_PGBOUNCER_MAX_CLIENT_CONN on the gateway container only. Works with + the module-created Aurora too: the pooler mints the IAM token itself and + renews it before it expires. EOT type = bool default = false diff --git a/tests/test_gateway/test_launch.py b/tests/test_gateway/test_launch.py index 964a7021416..6039aaafc2b 100644 --- a/tests/test_gateway/test_launch.py +++ b/tests/test_gateway/test_launch.py @@ -5,6 +5,7 @@ import textwrap import urllib.parse from pathlib import Path from typing import Final, cast +from unittest.mock import MagicMock, patch import pytest from uvicorn.importer import import_from_string @@ -13,7 +14,7 @@ from uvicorn.main import main as uvicorn_main import gateway.main from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv from litellm.proxy.db.db_url_settings import DatabaseURLSettings -from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings +from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings DB_ENV: Final = { "DATABASE_HOST": "db.internal", @@ -66,7 +67,13 @@ def _query(url: str) -> dict[str, str]: @pytest.fixture def password_env(monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: - for var in ("DATABASE_URL", "IAM_TOKEN_DB_AUTH", "AZURE_POSTGRESQL_AUTH", "DATABASE_HOST_READ_REPLICA"): + for var in ( + "DATABASE_URL", + "IAM_TOKEN_DB_AUTH", + "AZURE_POSTGRESQL_AUTH", + "DATABASE_HOST_READ_REPLICA", + PGBOUNCER_POOLED_ENV_VAR, + ): monkeypatch.setenv(var, "") monkeypatch.delenv(var) for var, value in DB_ENV.items(): @@ -74,6 +81,12 @@ def password_env(monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: return dict(DB_ENV) +def _minted_iam_token(token: str): + rds: Final = MagicMock() + rds.generate_db_auth_token.return_value = token + return patch("boto3.client", return_value=rds) + + def _uvicorn_params(argv: tuple[str, ...]) -> dict[str, object]: return uvicorn_main.make_context("uvicorn", list(argv)).params @@ -109,16 +122,23 @@ class TestPoolDatabaseUrl: assert isinstance(outcome, PgBouncerError) assert "DATABASE_URL" in outcome.reason - def test_token_auth_is_refused(self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + def test_token_auth_hands_the_workers_the_pool_user_not_the_token( + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ): monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") - environ: Final = {"DATABASE_URL": "postgresql://litellm:token@db.internal:5432/litellm"} - outcome: Final = pool_database_url( - DatabaseURLSettings.from_env(), - PgBouncerSettings(enabled=True, port=_free_port(), binary=str(_fake_pooler(tmp_path))), - environ, - ) - assert isinstance(outcome, PgBouncerError) - assert "IAM_TOKEN_DB_AUTH" in outcome.reason + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + port: Final = _free_port() + environ: Final = {"DATABASE_URL": "postgresql://litellm:MINTED_TOKEN@db.internal:5432/litellm"} + with _minted_iam_token("MINTED_TOKEN"): + outcome: Final = pool_database_url( + DatabaseURLSettings.from_env(), + PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path))), + environ, + ) + assert isinstance(outcome, str), outcome + pooled: Final = urllib.parse.urlsplit(outcome) + assert (pooled.username, pooled.hostname, pooled.port) == ("litellm_pgbouncer", "127.0.0.1", port) + assert "MINTED_TOKEN" not in outcome class TestMain: @@ -134,14 +154,39 @@ class TestMain: main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv))) pooled: Final = os.environ["DATABASE_URL"] - assert urllib.parse.urlsplit(pooled).netloc == f"litellm_pool:p%40ss@127.0.0.1:{port}" + assert urllib.parse.urlsplit(pooled).hostname == "127.0.0.1" + assert urllib.parse.urlsplit(pooled).port == port + assert urllib.parse.urlsplit(pooled).username == "litellm_pgbouncer" + assert "p%40ss" not in pooled assert _query(pooled)["pgbouncer"] == "true" assert _uvicorn_params(served[0])["timeout_keep_alive"] == 75 DatabaseURLSettings.from_env().apply_to_env() - worker_url: Final = os.environ["DATABASE_URL"] - assert urllib.parse.urlsplit(worker_url).netloc == f"litellm_pool:p%40ss@127.0.0.1:{port}" - assert _query(worker_url)["pgbouncer"] == "true" + assert urllib.parse.urlsplit(os.environ["DATABASE_URL"]).netloc == urllib.parse.urlsplit(pooled).netloc + assert _query(os.environ["DATABASE_URL"])["pgbouncer"] == "true" + + def test_iam_workers_keep_the_loopback_url_instead_of_minting_their_own( + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ): + port: Final = _free_port() + monkeypatch.delenv("DATABASE_PASSWORD") + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true") + monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", str(port)) + monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(tmp_path))) + served: Final[list[tuple[str, ...]]] = [] + with _minted_iam_token("SUPERVISOR_TOKEN"): + main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv))) + pooled: Final = os.environ["DATABASE_URL"] + assert urllib.parse.urlsplit(pooled).netloc.endswith(f"@127.0.0.1:{port}") + assert "SUPERVISOR_TOKEN" not in pooled + assert os.environ[PGBOUNCER_POOLED_ENV_VAR] == "true" + assert len(served) == 1 + + with _minted_iam_token("WORKER_TOKEN"): + DatabaseURLSettings.from_env().apply_to_env() + assert os.environ["DATABASE_URL"] == pooled def test_a_pooler_that_cannot_start_stops_the_gateway_before_uvicorn( self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path diff --git a/tests/test_litellm/proxy/db/test_pgbouncer.py b/tests/test_litellm/proxy/db/test_pgbouncer.py index da1d4c91f09..7b5a0bf10f9 100644 --- a/tests/test_litellm/proxy/db/test_pgbouncer.py +++ b/tests/test_litellm/proxy/db/test_pgbouncer.py @@ -1,4 +1,6 @@ +import base64 import configparser +import json import logging import os import signal @@ -9,8 +11,10 @@ import tempfile import textwrap import time import urllib.parse +from collections import deque from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Final, cast @@ -18,16 +22,24 @@ import pytest from litellm._logging import verbose_proxy_logger from litellm.proxy.db.pgbouncer import ( + PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerPlan, PgBouncerProcess, PgBouncerSettings, + PgBouncerTokenRefresher, + PgBouncerTokenSource, + database_url_is_pooled, + export_pooled_database_url, + install_pgbouncer_token, pgbouncer_version, plan_pgbouncer, start_in_container_pgbouncer, unix_socket_path, - write_pgbouncer_files, + write_pgbouncer_ini, + write_userlist, ) +from litellm.proxy.db.token_auth import AzureEntraTokenAuth, IAMEndpoint UPSTREAM: Final = ( "postgresql://app:p%40ss%27w@db.internal:5433/litellm" @@ -55,17 +67,24 @@ def _query(url: str) -> dict[str, str]: class TestPlanPgBouncer: - def test_upstream_credentials_and_timeouts_move_into_the_pgbouncer_config(self): + def test_upstream_route_and_timeouts_move_into_the_pgbouncer_config_without_the_password(self): ini: Final = _ini(_plan()) assert ini["databases"]["litellm"] == ( - "host='db.internal' port=5433 dbname='litellm' user='app' password='p@ss''w' " + "host='db.internal' port=5433 dbname='litellm' user='app' " "connect_query='SET statement_timeout TO ''7000''; SET lock_timeout TO ''3000'''" ) - assert _plan().userlist == '"app" "p@ss\'w"\n' + + def test_the_auth_file_holds_the_upstream_password_and_the_pool_users_own(self): + plan: Final = _plan() + assert plan.upstream_password == "p@ss'w" + assert plan.userlist("p@ss'w") == f'"app" "p@ss\'w"\n"litellm_pgbouncer" "{plan.pool_password}"\n' + + def test_a_token_with_quotes_is_escaped_the_way_pgbouncer_reads_it(self): + assert _plan().userlist('to"ken').startswith('"app" "to""ken"\n') def test_an_upstream_without_a_port_is_reached_on_the_postgres_default(self): ini: Final = _ini(_plan("postgresql://app:pw@db/litellm")) - assert ini["databases"]["litellm"] == "host='db' port=5432 dbname='litellm' user='app' password='pw'" + assert ini["databases"]["litellm"] == "host='db' port=5432 dbname='litellm' user='app'" def test_pool_is_sized_from_settings_in_transaction_mode(self): pgb: Final = _ini(_plan())["pgbouncer"] @@ -79,20 +98,24 @@ class TestPlanPgBouncer: assert pgb["auth_file"] == "/run/pgb/userlist.txt" assert pgb["unix_socket_dir"] == "/run/pgb" - def test_the_app_user_can_read_the_pgbouncer_console(self): - assert _ini(_plan())["pgbouncer"]["stats_users"] == "app" + def test_the_pool_user_can_read_the_pgbouncer_console(self): + assert _ini(_plan())["pgbouncer"]["stats_users"] == "litellm_pgbouncer" - @pytest.mark.parametrize("user", ["app,admin", "app%20admin", "app%09admin"]) - def test_a_user_pgbouncer_would_split_into_several_console_users_is_refused(self, user: str): - outcome: Final = plan_pgbouncer(f"postgresql://{user}:pw@db/litellm", SETTINGS, Path("/run/pgb"), None) + def test_a_database_user_named_like_the_pool_user_is_refused(self): + outcome: Final = plan_pgbouncer( + "postgresql://litellm_pgbouncer:pw@db/litellm", SETTINGS, Path("/run/pgb"), None + ) assert isinstance(outcome, PgBouncerError) - assert "stats_users" in outcome.reason + assert "litellm_pgbouncer" in outcome.reason - def test_pooled_url_points_prisma_at_loopback_without_prepared_statements(self): - pooled: Final = urllib.parse.urlsplit(_plan().pooled_url) + def test_pooled_url_points_prisma_at_loopback_as_the_pool_user_without_prepared_statements(self): + plan: Final = _plan() + pooled: Final = urllib.parse.urlsplit(plan.pooled_url) assert (pooled.hostname, pooled.port, pooled.path) == ("127.0.0.1", 6543, "/litellm") - assert (pooled.username, pooled.password) == ("app", "p%40ss%27w") - assert _query(_plan().pooled_url) == { + assert (pooled.username, pooled.password) == ("litellm_pgbouncer", plan.pool_password) + assert len(plan.pool_password) >= 32 + assert "p%40ss" not in plan.pooled_url + assert _query(plan.pooled_url) == { "schema": "public", "connection_limit": "10", "pool_timeout": "20", @@ -118,6 +141,9 @@ class TestPlanPgBouncer: assert "server_tls_ca_file" not in pgb assert plan.ca_source is None + def test_every_plan_gets_its_own_pool_password(self): + assert _plan().pool_password != _plan().pool_password + def test_no_tls_params_default_to_prefer(self): assert _ini(_plan("postgresql://app:pw@db/litellm"))["pgbouncer"]["server_tls_sslmode"] == "prefer" @@ -138,15 +164,20 @@ class TestPlanPgBouncer: @pytest.mark.parametrize( "url", [ - "postgresql://app@db/litellm", "postgresql://app:pw@db", "postgresql://:pw@db/litellm", + "postgresql://app:pw@/litellm", ], ) - def test_urls_missing_forwardable_credentials_are_refused(self, url: str): + def test_urls_missing_a_route_are_refused(self, url: str): outcome: Final = plan_pgbouncer(url, SETTINGS, Path("/run/pgb"), None) assert isinstance(outcome, PgBouncerError) + def test_a_url_without_a_password_plans_for_a_token_to_be_installed_later(self): + plan: Final = _plan("postgresql://app@db/litellm") + assert plan.upstream_password is None + assert plan.userlist("minted").startswith('"app" "minted"\n') + def test_every_options_spelling_becomes_a_set_statement(self): options: Final = urllib.parse.quote("-c a=1 -cb=2 --c=3") ini: Final = _ini(_plan(f"postgresql://app:pw@db/litellm?options={options}")) @@ -167,12 +198,14 @@ class TestPlanPgBouncer: class TestWritePgBouncerFiles: def test_files_hold_the_plan_and_are_private_to_the_owner(self, tmp_path: Path): plan: Final = _plan("postgresql://app:pw@db/litellm") - ini_path: Final = write_pgbouncer_files(plan, tmp_path, None) + ini_path: Final = write_pgbouncer_ini(plan, tmp_path, None) assert isinstance(ini_path, Path), ini_path + userlist_path: Final = write_userlist(plan.userlist("pw"), tmp_path, None) assert ini_path == tmp_path / "pgbouncer.ini" + assert userlist_path == tmp_path / "userlist.txt" assert ini_path.read_text() == plan.ini - assert (tmp_path / "userlist.txt").read_text() == plan.userlist - for path in (ini_path, tmp_path / "userlist.txt"): + assert userlist_path.read_text() == plan.userlist("pw") + for path in (ini_path, userlist_path): assert stat.S_IMODE(path.stat().st_mode) == 0o600 assert not (tmp_path / "server-ca.pem").exists() @@ -185,7 +218,7 @@ class TestWritePgBouncerFiles: f"postgresql://app:pw@db/litellm?sslmode=verify-full&sslcert={bundle}", SETTINGS, runtime_dir, None ) assert isinstance(plan, PgBouncerPlan), plan - ini_path: Final = write_pgbouncer_files(plan, runtime_dir, None) + ini_path: Final = write_pgbouncer_ini(plan, runtime_dir, None) assert isinstance(ini_path, Path), ini_path ca_file: Final = Path(_ini(plan)["pgbouncer"]["server_tls_ca_file"]) assert ca_file.parent == runtime_dir @@ -199,11 +232,32 @@ class TestWritePgBouncerFiles: None, ) assert isinstance(plan, PgBouncerPlan), plan - outcome: Final = write_pgbouncer_files(plan, tmp_path, None) + outcome: Final = write_pgbouncer_ini(plan, tmp_path, None) assert isinstance(outcome, PgBouncerError) assert "missing.pem" in outcome.reason assert not (tmp_path / "pgbouncer.ini").exists() + def test_rewriting_the_userlist_replaces_it_whole_and_leaves_nothing_else_behind(self, tmp_path: Path): + write_userlist('"app" "first"\n', tmp_path, None) + with open(tmp_path / "userlist.txt", encoding="utf-8") as before_rewrite: + write_userlist('"app" "second"\n', tmp_path, None) + assert before_rewrite.read() == '"app" "first"\n' + assert (tmp_path / "userlist.txt").read_text() == '"app" "second"\n' + assert stat.S_IMODE((tmp_path / "userlist.txt").stat().st_mode) == 0o600 + assert sorted(path.name for path in tmp_path.iterdir()) == ["userlist.txt"] + + +class TestPooledUrlMarker: + def test_exporting_the_pooled_url_marks_it_for_the_workers(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv(PGBOUNCER_POOLED_ENV_VAR, "") + monkeypatch.delenv(PGBOUNCER_POOLED_ENV_VAR) + monkeypatch.setenv("DATABASE_URL", "postgresql://app:token@db/litellm") + assert not database_url_is_pooled() + export_pooled_database_url("postgresql://litellm_pgbouncer:pw@127.0.0.1:6432/litellm?pgbouncer=true") + assert os.environ["DATABASE_URL"] == "postgresql://litellm_pgbouncer:pw@127.0.0.1:6432/litellm?pgbouncer=true" + assert database_url_is_pooled() + assert PGBOUNCER_POOLED_ENV_VAR == "LITELLM_PGBOUNCER_POOLED_DATABASE_URL" + def _bound_port(sock: socket.socket) -> int: return cast(tuple[str, int], sock.getsockname())[1] @@ -222,20 +276,23 @@ def _fake_pooler( port_file: Path | None = None, bind_delay_seconds: float = 0.0, version_banner: str = "PgBouncer 1.25.2\nlibevent 2.1.13-stable", + auth_log: Path | None = None, ) -> Path: """An executable that listens like PgBouncer: on the TCP port first, then on ``.s.PGSQL.`` in the socket dir. Port and socket dir come from the ini it is given, else from ``port`` and ``tmp_path``. With ``port_file`` each start reads the port from that file instead. ``bind_delay_seconds`` holds the bind back, like a slow start. - ``--version`` prints ``version_banner``. + ``--version`` prints ``version_banner``. With ``auth_log`` it appends the + ``auth_file`` it reads at startup and on every SIGHUP, one line per read, + like PgBouncer loading its credentials. """ script: Final = tmp_path / "fake-pgbouncer" script.write_text( textwrap.dedent( f"""\ #!{sys.executable} - import configparser, os, pathlib, select, socket, sys, time + import configparser, os, pathlib, select, signal, socket, sys, time if sys.argv[1:] == ["--version"]: print({version_banner!r}) sys.exit(0) @@ -243,6 +300,12 @@ def _fake_pooler( sys.exit(3) ini = configparser.ConfigParser() ini.read(sys.argv[1:2]) + if not {auth_log is None!r}: + def load_auth_file(*_): + with open({str(auth_log)!r}, "a") as log: + log.write(repr(pathlib.Path(ini.get("pgbouncer", "auth_file")).read_text()) + "\\n") + load_auth_file() + signal.signal(signal.SIGHUP, load_auth_file) port = ini.getint("pgbouncer", "listen_port", fallback={port}) if not {port_file is None!r}: port = int(pathlib.Path({str(port_file)!r}).read_text()) @@ -511,6 +574,118 @@ class TestPgBouncerProcess: os.kill(pid, 0) +NOW: Final = datetime(2026, 9, 10, 12, 0, tzinfo=timezone.utc) +ENDPOINT: Final = IAMEndpoint(host="db", port="5432", user="app", name="litellm") + + +def _entra_jwt(expires_at: datetime) -> str: + payload: Final = base64.urlsafe_b64encode(json.dumps({"exp": int(expires_at.timestamp())}).encode()) + return f"aGVhZGVy.{payload.rstrip(b'=').decode()}.c2ln" + + +def _token_source(*tokens: str | Exception) -> PgBouncerTokenSource: + """A token source handing out ``tokens`` in order, raising the exceptions among them, then repeating the last.""" + pending: Final = deque(tokens) + + def provide() -> str: + outcome: Final = pending.popleft() if len(pending) > 1 else pending[0] + if isinstance(outcome, Exception): + raise outcome + return outcome + + return PgBouncerTokenSource(auth=AzureEntraTokenAuth(token_provider=provide), endpoint=ENDPOINT) + + +class TestPgBouncerTokenRefresher: + def _refresher( + self, + source: PgBouncerTokenSource, + installed: list[str], + install: Callable[[str], None] | None = None, + **timing: float, + ) -> PgBouncerTokenRefresher: + return PgBouncerTokenRefresher( + source, + install if install is not None else installed.append, + now=lambda: NOW.replace(tzinfo=None), + **timing, + ) + + def test_the_next_refresh_is_due_a_buffer_before_the_token_expires(self): + installed: Final[list[str]] = [] + token: Final = _entra_jwt(NOW + timedelta(hours=1)) + refresher: Final = self._refresher(_token_source(token), installed, buffer_seconds=180) + assert refresher.refresh() == 3600 - 180 + assert installed == [token] + + def test_a_token_whose_expiry_cannot_be_read_is_refreshed_on_the_fallback_interval(self): + installed: Final[list[str]] = [] + refresher: Final = self._refresher(_token_source("opaque token"), installed, fallback_seconds=600) + assert refresher.refresh() == 600 + assert installed == ["opaque token"] + + def test_a_token_already_inside_the_buffer_is_refreshed_after_the_retry_delay(self): + token: Final = _entra_jwt(NOW + timedelta(seconds=100)) + refresher: Final = self._refresher(_token_source(token), [], buffer_seconds=180, retry_seconds=30) + assert refresher.refresh() == 30 + + def test_the_token_reaches_the_auth_file_in_wire_form_not_url_encoded(self): + installed: Final[list[str]] = [] + self._refresher(_token_source("to ken/with+odd=chars"), installed).refresh() + assert installed == ["to ken/with+odd=chars"] + + def test_a_failed_mint_is_reported_and_installs_nothing(self): + installed: Final[list[str]] = [] + outcome: Final = self._refresher(_token_source(RuntimeError("no credential")), installed).refresh() + assert isinstance(outcome, PgBouncerError) + assert "Azure Entra token" in outcome.reason + assert "no credential" in outcome.reason + assert installed == [] + + def test_a_token_pgbouncer_cannot_hold_is_refused(self): + installed: Final[list[str]] = [] + outcome: Final = self._refresher(_token_source("x" * 2048), installed).refresh() + assert isinstance(outcome, PgBouncerError) + assert "2047" in outcome.reason + assert installed == [] + + def test_an_auth_file_that_cannot_be_written_is_reported_not_raised(self): + def refuse(_: str) -> None: + raise PermissionError("read-only runtime dir") + + outcome: Final = self._refresher(_token_source("token"), [], install=refuse).refresh() + assert isinstance(outcome, PgBouncerError) + assert "read-only runtime dir" in outcome.reason + + def test_start_fails_when_the_first_token_cannot_be_minted_and_schedules_nothing(self): + installed: Final[list[str]] = [] + refresher: Final = self._refresher( + _token_source(RuntimeError("no credential"), "later"), installed, fallback_seconds=0.05 + ) + assert isinstance(refresher.start(), PgBouncerError) + time.sleep(0.3) + assert installed == [] + + def test_a_failed_renewal_keeps_the_previous_token_until_the_retry_succeeds(self, caplog: pytest.LogCaptureFixture): + installed: Final[list[str]] = [] + refresher: Final = self._refresher( + _token_source("first", RuntimeError("blip"), "third"), + installed, + fallback_seconds=0.05, + retry_seconds=0.05, + ) + with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): + assert refresher.start() is None + assert installed == ["first"] + assert _wait_until(lambda: "third" in installed) + assert installed[:2] == ["first", "third"] + assert any("keeps its current Azure Entra token" in record.message for record in caplog.records) + refresher.stop() + settled: Final = len(installed) + time.sleep(0.3) + assert len(installed) == settled + + def _runtime_dir_listening_on(port: int) -> Path: matches: Final = tuple( ini.parent @@ -524,10 +699,21 @@ def _runtime_dir_listening_on(port: int) -> Path: class TestStartInContainerPgBouncer: def test_returns_the_loopback_url_once_the_pooler_listens(self, tmp_path: Path): port: Final = _free_port() - settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) + auth_log: Final = tmp_path / "auth.log" + binary: Final = _fake_pooler(tmp_path, port, auth_log=auth_log) + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) pooled: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm?connection_limit=5") - assert pooled == f"postgresql://app:pw@127.0.0.1:{port}/litellm?connection_limit=5&pgbouncer=true" + assert isinstance(pooled, str), pooled + parsed: Final = urllib.parse.urlsplit(pooled) + assert (parsed.username, parsed.hostname, parsed.port, parsed.path) == ( + "litellm_pgbouncer", + "127.0.0.1", + port, + "/litellm", + ) + assert _query(pooled) == {"connection_limit": "5", "pgbouncer": "true"} assert _listening(port) + assert auth_log.read_text() == repr(f'"app" "pw"\n"litellm_pgbouncer" "{parsed.password}"\n') + "\n" @pytest.mark.filterwarnings("ignore:This process .* is multi-threaded:DeprecationWarning") def test_a_forked_worker_exiting_leaves_the_pooler_and_its_files_to_the_parent(self, tmp_path: Path): @@ -561,21 +747,79 @@ class TestStartInContainerPgBouncer: def test_a_bad_upstream_url_is_reported_without_starting_anything(self, tmp_path: Path): port: Final = _free_port() settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) - outcome: Final = start_in_container_pgbouncer(settings, "postgresql://app@db/litellm") + outcome: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db") assert isinstance(outcome, PgBouncerError) assert not _listening(port) - def test_token_auth_is_refused_without_starting_anything(self, tmp_path: Path): + def test_a_passwordless_url_without_token_auth_is_refused_without_starting_anything(self, tmp_path: Path): port: Final = _free_port() settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) - outcome: Final = start_in_container_pgbouncer( - settings, "postgresql://app:pw@db/litellm", token_auth_enabled=True - ) + outcome: Final = start_in_container_pgbouncer(settings, "postgresql://app@db/litellm") assert isinstance(outcome, PgBouncerError) assert "IAM_TOKEN_DB_AUTH" in outcome.reason - assert "AZURE_POSTGRESQL_AUTH" in outcome.reason assert not _listening(port) + def test_token_auth_mints_the_first_token_into_the_auth_file_before_the_pooler_starts(self, tmp_path: Path): + port: Final = _free_port() + auth_log: Final = tmp_path / "auth.log" + binary: Final = _fake_pooler(tmp_path, port, auth_log=auth_log) + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) + token: Final = _entra_jwt(datetime.now(tz=timezone.utc) + timedelta(hours=1)) + pooled: Final = start_in_container_pgbouncer( + settings, + "postgresql://app:stale-token@db/litellm", + token_auth=AzureEntraTokenAuth(token_provider=lambda: token), + ) + assert isinstance(pooled, str), pooled + parsed: Final = urllib.parse.urlsplit(pooled) + assert parsed.username == "litellm_pgbouncer" + assert token not in pooled + assert _listening(port) + assert auth_log.read_text() == repr(f'"app" "{token}"\n"litellm_pgbouncer" "{parsed.password}"\n') + "\n" + + def test_a_first_token_that_cannot_be_minted_is_reported_without_starting_anything(self, tmp_path: Path): + port: Final = _free_port() + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) + + def fail() -> str: + raise RuntimeError("no Azure credential") + + outcome: Final = start_in_container_pgbouncer( + settings, "postgresql://app@db/litellm", token_auth=AzureEntraTokenAuth(token_provider=fail) + ) + assert isinstance(outcome, PgBouncerError) + assert "no Azure credential" in outcome.reason + assert not _listening(port) + + def test_a_renewed_token_is_written_and_picked_up_by_the_running_and_by_a_restarted_pooler(self, tmp_path: Path): + port: Final = _free_port() + auth_log: Final = tmp_path / "auth.log" + plan: Final = plan_pgbouncer( + "postgresql://app@db/litellm", PgBouncerSettings(enabled=True, port=port), tmp_path, None + ) + assert isinstance(plan, PgBouncerPlan), plan + ini_path: Final = write_pgbouncer_ini(plan, tmp_path, None) + write_userlist(plan.userlist("first"), tmp_path, None) + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port, auth_log=auth_log)), str(ini_path)), + port=port, + socket_path=unix_socket_path(tmp_path, port), + restart_delay_seconds=0.1, + ) + assert pooler.start() is None + first_pid: Final = pooler.pid + assert first_pid is not None + install_pgbouncer_token(plan, tmp_path, None, pooler, "second") + assert _wait_until(lambda: auth_log.read_text().count("\n") == 2) + os.kill(first_pid, signal.SIGKILL) + assert _wait_until(lambda: pooler.pid not in (None, first_pid) and _listening(port)) + pooler.stop() + assert auth_log.read_text().splitlines() == [ + repr(plan.userlist("first")), + repr(plan.userlist("second")), + repr(plan.userlist("second")), + ] + def test_a_pgbouncer_that_survives_a_failed_tcp_bind_is_refused_without_starting(self, tmp_path: Path): port: Final = _free_port() binary: Final = _fake_pooler(tmp_path, port, version_banner="PgBouncer 1.18.1\nlibevent 2.1.12-stable") @@ -590,9 +834,9 @@ class TestStartInContainerPgBouncer: port: Final = _free_port() binary: Final = _fake_pooler(tmp_path, port, version_banner="PgBouncer 1.19.0") settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) - assert start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm") == ( - f"postgresql://app:pw@127.0.0.1:{port}/litellm?pgbouncer=true" - ) + pooled: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm") + assert isinstance(pooled, str), pooled + assert urllib.parse.urlsplit(pooled).port == port assert _listening(port) diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 1287cf23157..646596c5b88 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -1,5 +1,6 @@ import datetime as real_datetime import smtplib +from typing import Final import pytest from fastapi import HTTPException @@ -8,7 +9,7 @@ from litellm.caching.caching import DualCache from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth -from litellm.proxy.utils import ProxyLogging +from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.types.guardrails import GuardrailEventHooks @@ -2207,3 +2208,49 @@ async def test_post_call_failure_hook_redacts_traceback_before_callbacks(monkeyp assert recorder.received_traceback is not None assert provider_key not in recorder.received_traceback assert "REDACTED" in recorder.received_traceback + + +class TestPrismaClientTokenAuthBehindThePool: + """Behind the in-container pool the supervisor renews the writer's database + token and hands the workers a loopback URL with a static password, so the + writer wrapper must not run its own refresh loop. The reader is not pooled + and keeps refreshing its own token.""" + + UPSTREAM: Final = "postgresql://litellm:TOKEN@db.internal:5432/litellm" + READER: Final = "postgresql://litellm:TOKEN@reader.internal:5432/litellm" + + def _client(self, monkeypatch: pytest.MonkeyPatch, pooled: bool) -> PrismaClient: + from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR + + monkeypatch.delenv("AZURE_POSTGRESQL_AUTH", raising=False) + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + monkeypatch.setenv("DATABASE_URL", self.UPSTREAM) + monkeypatch.setenv("DATABASE_URL_READ_REPLICA", self.READER) + if pooled: + monkeypatch.setenv(PGBOUNCER_POOLED_ENV_VAR, "true") + else: + monkeypatch.delenv(PGBOUNCER_POOLED_ENV_VAR, raising=False) + rds: Final = MagicMock() + rds.generate_db_auth_token.return_value = "TOKEN" + with patch("boto3.client", return_value=rds): + return PrismaClient(database_url=self.UPSTREAM, proxy_logging_obj=MagicMock(spec=ProxyLogging)) + + def test_a_pooled_writer_leaves_token_refresh_to_the_pooler_while_the_reader_keeps_its_own( + self, monkeypatch: pytest.MonkeyPatch + ): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + client = self._client(monkeypatch, pooled=True) + assert isinstance(client.db, RoutingPrismaWrapper) + assert client.db.writer.iam_token_db_auth is False + assert client.db.reader.iam_token_db_auth is True + assert client.token_auth is not None + + def test_an_unpooled_writer_still_refreshes_its_own_token(self, monkeypatch: pytest.MonkeyPatch): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + client = self._client(monkeypatch, pooled=False) + assert isinstance(client.db, RoutingPrismaWrapper) + assert client.db.writer.iam_token_db_auth is True + assert client.db.reader.iam_token_db_auth is True From 033f2e5e2a651fed59faf6478fb18314e487738b Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 14:19:08 -0700 Subject: [PATCH 097/126] feat(wandb): default unmapped W&B models to reasoning-capable W&B's serverless catalog grows faster than the registry names it, so a model they ship today resolves as non-reasoning here until someone edits the cost map, and the caller's reasoning_effort is dropped or rejected. Add a wandb-reasoning-baseline capability rule to fallback_generalizations so any wandb/ id the map has not described defaults to supports_reasoning. Rules lose to exact entries, so mapped non-reasoning models such as wandb/meta-llama/Llama-3.1-8B-Instruct are unaffected. The rule carries no mode and no pricing, so cost stays on the standard unpriced behavior and the deployment does not read as catalog-mapped to the router's reasoning-effort resolver. Claude-Session: https://claude.ai/code/session_01A6SkwJdfZUmkzfUkrEkqX8 --- ...odel_prices_and_context_window_backup.json | 8 +++ model_prices_and_context_window.json | 8 +++ .../test_fallback_generalizations.py | 60 +++++++++++++++++++ .../wandb/test_wandb_chat_transformation.py | 26 +++++++- 4 files changed, 101 insertions(+), 1 deletion(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1c0bd32d782..b7726290f0e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -57258,6 +57258,14 @@ "model_info": { "supports_mid_conversation_system": true } + }, + { + "name": "wandb-reasoning-baseline", + "pattern": "^wandb/", + "description": "Any Weights & Biases Inference model id, anchored to the wandb/ namespace so only that provider's ids match. W&B's serverless catalog is reasoning-first and grows faster than this registry names it, so an id the map has not described yet is treated as reasoning-capable and keeps the caller's reasoning_effort instead of dropping it or raising UnsupportedParamsError. Rules lose to exact entries, so a mapped non-reasoning model such as wandb/meta-llama/Llama-3.1-8B-Instruct is unaffected. Carries no mode and no pricing, so cost stays on the standard unpriced behavior and the deployment does not read as catalog-mapped to the router's reasoning-effort resolver.", + "model_info": { + "supports_reasoning": true + } } ] }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1c0bd32d782..b7726290f0e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -57258,6 +57258,14 @@ "model_info": { "supports_mid_conversation_system": true } + }, + { + "name": "wandb-reasoning-baseline", + "pattern": "^wandb/", + "description": "Any Weights & Biases Inference model id, anchored to the wandb/ namespace so only that provider's ids match. W&B's serverless catalog is reasoning-first and grows faster than this registry names it, so an id the map has not described yet is treated as reasoning-capable and keeps the caller's reasoning_effort instead of dropping it or raising UnsupportedParamsError. Rules lose to exact entries, so a mapped non-reasoning model such as wandb/meta-llama/Llama-3.1-8B-Instruct is unaffected. Carries no mode and no pricing, so cost stays on the standard unpriced behavior and the deployment does not read as catalog-mapped to the router's reasoning-effort resolver.", + "model_info": { + "supports_reasoning": true + } } ] }, diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index b1e8163b91d..56552ec20ab 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -571,3 +571,63 @@ def test_shipped_mid_conversation_gate_on_bedrock_ids(shipped_cost_map): ): matched = match_capability_generalizations(unflagged) assert matched is None or not matched.get("supports_mid_conversation_system"), unflagged + + +def test_shipped_rules_flag_unmapped_wandb_ids_as_reasoning(shipped_cost_map): + """W&B ships reasoning models faster than the registry names them, so an unmapped + wandb id resolves as reasoning-capable and its reasoning_effort survives instead of + being dropped. The rule carries no mode and no pricing, so cost stays on the standard + unpriced behavior and the deployment does not read as catalog-mapped.""" + model = "wandb/zai-org/GLM-6-Turbo" + assert model not in litellm.model_cost + + info = litellm.get_model_info(model, custom_llm_provider="wandb") + assert info["litellm_provider"] == "wandb" + assert info["supports_reasoning"] is True + assert info.get("mode") is None + assert not info.get("input_cost_per_token") + assert not info.get("output_cost_per_token") + + assert litellm.supports_reasoning(model="zai-org/GLM-6-Turbo", custom_llm_provider="wandb") is True + + +def test_shipped_wandb_rule_loses_to_mapped_non_reasoning_entries(shipped_cost_map): + """The whole point of a fallback is that it only fills gaps. A wandb model the map + describes as non-reasoning must stay non-reasoning, otherwise the rule silently + re-introduces the blanket supports_reasoning it exists to avoid.""" + for model in ( + "meta-llama/Llama-3.1-8B-Instruct", + "microsoft/Phi-4-mini-instruct", + "moonshotai/Kimi-K2-Instruct", + "Qwen/Qwen3-Coder-480B-A35B-Instruct", + ): + assert f"wandb/{model}" in litellm.model_cost, model + assert litellm.supports_reasoning(model=model, custom_llm_provider="wandb") is False, model + + +def test_shipped_wandb_rule_is_anchored_to_the_wandb_namespace(shipped_cost_map): + """``^wandb/`` is anchored, so it cannot leak onto another provider's ids.""" + assert match_capability_generalizations("wandb/some-new-model") == {"supports_reasoning": True} + for foreign in ("openai/some-new-model", "notwandb/some-new-model", "together_ai/wandb/some-new-model"): + matched = match_capability_generalizations(foreign) + assert matched is None or not matched.get("supports_reasoning"), foreign + + +def test_shipped_wandb_rule_keeps_reasoning_effort_on_an_unmapped_model(shipped_cost_map): + """End to end through the provider config: the gate WandbConfig applies reads the + rule, so reasoning_effort is advertised and survives get_optional_params rather than + raising UnsupportedParamsError.""" + model = "zai-org/GLM-6-Turbo" + assert f"wandb/{model}" not in litellm.model_cost + + supported = litellm.get_supported_openai_params(model=f"wandb/{model}") + assert supported is not None + assert "reasoning_effort" in supported + + optional_params = litellm.utils.get_optional_params( + model=model, + custom_llm_provider="wandb", + reasoning_effort="medium", + drop_params=False, + ) + assert optional_params["reasoning_effort"] == "medium" diff --git a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py index 481a52bebad..dd0d1bdbb9d 100644 --- a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py +++ b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py @@ -249,7 +249,6 @@ class TestWandbConfig: "model,explicit_false", [ ("meta-llama/Llama-3.1-8B-Instruct", False), - ("unknown-model", False), ("openai/gpt-oss-20b", True), ], ) @@ -290,3 +289,28 @@ class TestWandbConfig: supported_params = litellm.get_supported_openai_params(model=f"wandb/{model}") assert supported_params is not None assert "reasoning_effort" not in supported_params + + @pytest.mark.respx() + def test_wandb_completion_keeps_reasoning_effort_for_an_unregistered_model( + self, wandb_test_config, wandb_request_mock: respx.Route + ): + """A wandb id the registry has not named yet resolves through the + wandb-reasoning-baseline fallback generalization, so its reasoning_effort reaches + the provider instead of raising. W&B adds reasoning models faster than this + registry names them, and an exact entry still wins wherever one exists.""" + model: Final = "zai-org/GLM-6-Turbo" + assert f"wandb/{model}" not in litellm.model_cost + + completion( + model=f"wandb/{model}", + messages=[{"role": "user", "content": "Hello"}], + api_key="fake-wandb-key", + api_base="https://api.inference.wandb.ai/v1", + reasoning_effort="medium", + drop_params=False, + ) + + assert wandb_request_mock.call_count == 1 + request_body = json.loads(wandb_request_mock.calls[0].request.content) + assert request_body["model"] == model + assert request_body["reasoning_effort"] == "medium" From 7a1178a8595617d29453b20bd2e76fc629476999 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 14:36:22 -0700 Subject: [PATCH 098/126] fix(utils): stop model registration from shadowing capability rules Router writes every configured deployment into litellm.model_cost, and a deployment that declares no model_info lands there as an empty entry. An exact entry ends the model-info lookup ladder before the fallback generalizations are consulted, so that empty entry made the rules inert for the model: configuring one on a proxy stripped the capabilities the same model resolves to off-proxy. Seed a new registration from the capability rules its key matches. The caller's own model_info still wins field by field, so an explicit supports_reasoning: false on the deployment keeps overriding the rule. Claude-Session: https://claude.ai/code/session_01A6SkwJdfZUmkzfUkrEkqX8 --- litellm/utils.py | 5 +- .../test_fallback_generalizations.py | 46 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index b4a2a29239b..69ee730ff30 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3108,7 +3108,10 @@ def register_model( existing_model = cast(dict, builtin_model_info) model_cost_key = existing_model["key"] else: - existing_model = {} + # An exact entry ends the lookup ladder before the capability rules are + # consulted, so seed from them: otherwise registering an unmapped model + # shadows the very defaults it would have resolved to unregistered. + existing_model = dict(match_capability_generalizations(_key_str) or {}) # mutable-ok: merge target model_cost_key = key builtin_entry = _resolve_builtin_model_cost_entry(key=_key_str, provider=provider) if builtin_entry is not None: diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index 56552ec20ab..b0220a36054 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -631,3 +631,49 @@ def test_shipped_wandb_rule_keeps_reasoning_effort_on_an_unmapped_model(shipped_ drop_params=False, ) assert optional_params["reasoning_effort"] == "medium" + + +def test_router_registration_does_not_shadow_shipped_rules(shipped_cost_map): + """Regression: Router writes every configured deployment into ``litellm.model_cost``, + and an exact entry ends the lookup ladder before the rules are consulted. Registering + an unmapped model has to carry the rule defaults forward, or configuring a model on a + proxy silently strips the capabilities the same model resolves to off-proxy.""" + from litellm import Router + + unmapped_wandb = "wandb/zai-org/GLM-6-Turbo" + unmapped_claude = "anthropic/claude-opus-9" + assert unmapped_wandb not in litellm.model_cost + assert unmapped_claude not in litellm.model_cost + + Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": name, "api_key": "fake"}} + for name in (unmapped_wandb, unmapped_claude) + ] + ) + + assert unmapped_wandb in litellm.model_cost + assert unmapped_claude in litellm.model_cost + assert litellm.supports_reasoning(model="zai-org/GLM-6-Turbo", custom_llm_provider="wandb") is True + assert litellm.supports_reasoning(model="claude-opus-9", custom_llm_provider="anthropic") is True + + +def test_deployment_model_info_beats_the_seeded_rule_defaults(shipped_cost_map): + """Seeding a registration from the rules is a floor, not an override: an explicit + model_info on the deployment still wins, so a non-reasoning model can be configured + under a reasoning-first namespace.""" + from litellm import Router + + model = "wandb/some-org/NoThink-1" + Router( + model_list=[ + { + "model_name": model, + "litellm_params": {"model": model, "api_key": "fake"}, + "model_info": {"supports_reasoning": False}, + } + ] + ) + + assert litellm.model_cost[model]["supports_reasoning"] is False + assert litellm.supports_reasoning(model="some-org/NoThink-1", custom_llm_provider="wandb") is False From 61b0def8670c7edce4c4c9bd6a324b3827deb565 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Thu, 10 Sep 2026 16:51:37 -0700 Subject: [PATCH 099/126] test(ocr): isolate native Rust bridge contracts (#40410) * test(rust): add retained callback suite as expected failures * fix(tests): narrow retained callback xfails * test(rust): clarify retained callback contracts * test(ocr): clarify retained Rust contracts * test(ocr): restore guardrail contracts * test(ocr): require Rust file input parity * test(ocr): isolate native bridge contracts * fix(ci): repair Rust dispatch and OSV checks * test(ocr): assert explicit backend dispatch --- .github/workflows/test-rust.yml | 28 + tests/test_litellm_rust/README.md | 13 + tests/test_litellm_rust/conftest.py | 134 ++++- tests/test_litellm_rust/ocr/__init__.py | 1 + tests/test_litellm_rust/ocr/test_callbacks.py | 513 ++++++++++++++++++ tests/test_litellm_rust/ocr/test_dispatch.py | 44 ++ .../test_litellm_rust/ocr/test_guardrails.py | 74 +++ tests/test_litellm_rust/ocr/test_requests.py | 434 +++++++++++++++ tests/test_litellm_rust/support/__init__.py | 1 + .../support/callback_recorder.py | 101 ++++ .../support/recording_server.py | 108 ++++ tests/test_litellm_rust/support/requests.py | 59 ++ tests/test_litellm_rust/test_ocr.py | 61 ++- 13 files changed, 1533 insertions(+), 38 deletions(-) create mode 100644 tests/test_litellm_rust/README.md create mode 100644 tests/test_litellm_rust/ocr/__init__.py create mode 100644 tests/test_litellm_rust/ocr/test_callbacks.py create mode 100644 tests/test_litellm_rust/ocr/test_dispatch.py create mode 100644 tests/test_litellm_rust/ocr/test_guardrails.py create mode 100644 tests/test_litellm_rust/ocr/test_requests.py create mode 100644 tests/test_litellm_rust/support/__init__.py create mode 100644 tests/test_litellm_rust/support/callback_recorder.py create mode 100644 tests/test_litellm_rust/support/recording_server.py create mode 100644 tests/test_litellm_rust/support/requests.py diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index c6901411167..17b6481a2bf 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -4,8 +4,22 @@ on: push: paths: - "litellm-rust/**" + - "litellm/rust_bridge/**" + - "tests/test_litellm_rust/**" + - "litellm/integrations/custom_logger.py" + - "litellm/litellm_core_utils/litellm_logging.py" + - "litellm/litellm_core_utils/logging_worker.py" + - "litellm/proxy/guardrails/**" + - "litellm/utils.py" + - "litellm/ocr/**" + - "litellm/llms/base_llm/ocr/**" + - "litellm/llms/custom_httpx/llm_http_handler.py" + - "tests/test_litellm/ocr/**" + - "tests/test_litellm/conftest.py" + - "Makefile" - ".cargo/**" - "pyproject.toml" + - "uv.lock" - "rust-toolchain.toml" - ".github/actions/setup-uv-with-retries/**" - ".github/scripts/smoke_test_native_wheel.py" @@ -20,8 +34,22 @@ on: - "litellm_**" paths: - "litellm-rust/**" + - "litellm/rust_bridge/**" + - "tests/test_litellm_rust/**" + - "litellm/integrations/custom_logger.py" + - "litellm/litellm_core_utils/litellm_logging.py" + - "litellm/litellm_core_utils/logging_worker.py" + - "litellm/proxy/guardrails/**" + - "litellm/utils.py" + - "litellm/ocr/**" + - "litellm/llms/base_llm/ocr/**" + - "litellm/llms/custom_httpx/llm_http_handler.py" + - "tests/test_litellm/ocr/**" + - "tests/test_litellm/conftest.py" + - "Makefile" - ".cargo/**" - "pyproject.toml" + - "uv.lock" - "rust-toolchain.toml" - ".github/actions/setup-uv-with-retries/**" - ".github/scripts/smoke_test_native_wheel.py" diff --git a/tests/test_litellm_rust/README.md b/tests/test_litellm_rust/README.md new file mode 100644 index 00000000000..4c117fb846b --- /dev/null +++ b/tests/test_litellm_rust/README.md @@ -0,0 +1,13 @@ +# Rust OCR bridge tests + +This suite covers OCR requests through LiteLLM's compiled Rust extension. OCR behavior tests live under `ocr/`; reusable OCR request, callback, and recording-server fixtures live under `support/` + +A test name identifies the OCR entrypoint or callback under test and its expected observable result. Parameter IDs state the execution mode or credential case. Keep multiple assertions together only when they prove one request, mutation, failure, or callback lifecycle behavior. Record callback observations and assert them after the callback returns because production logging can swallow callback exceptions + +`ocr/test_requests.py` covers provider payloads, file preparation, endpoint and credential resolution, normalized responses, errors, timeouts, and Azure token-provider behavior. `ocr/test_callbacks.py` covers OCR callback inputs, mutations, ordering, context, failure handling, concurrency, and cleanup. `ocr/test_guardrails.py` covers OCR post-call blocking and response replacement. These contract modules call the Rust bridge directly. `ocr/test_dispatch.py` has the single public API dispatch test, covering enabled native dispatch and disabled Python dispatch. `test_ocr.py` is a strict smoke test of the compiled Rust OCR transport + +Run `make test-rust-extension` as the acceptance command. It builds a fresh wheel, installs that wheel into a temporary environment, requires `LITELLM_RUST=1`, and runs this suite with isolated Python imports + +Collection fails when `LITELLM_RUST=1` is set but the compiled `_native` module cannot be imported. The autouse fixture isolates callback and configuration state but does not select a backend. Native contract tests call `litellm.rust_bridge.ocr` directly, while the strict dispatch test explicitly enables and disables Rust and records which OCR entrypoint runs + +The OCR contract modules are non-strict expected failures until the retained callback implementation from #40070 lands. The public dispatch test remains strict. Passing contract cases appear as XPASS so staging coverage stays visible diff --git a/tests/test_litellm_rust/conftest.py b/tests/test_litellm_rust/conftest.py index 02d274bb405..b0c75d9d2f5 100644 --- a/tests/test_litellm_rust/conftest.py +++ b/tests/test_litellm_rust/conftest.py @@ -1,24 +1,132 @@ +import asyncio import os +from collections.abc import AsyncIterator, Generator, Iterator +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack, contextmanager +from types import ModuleType +from typing import Final, cast import pytest +import pytest_asyncio + +import litellm +from litellm import utils +from litellm.litellm_core_utils import litellm_logging +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.rust_bridge.configuration import ( # pyright: ignore[reportPrivateUsage] # preserve raw configuration state in test isolation + _CONFIGURATION, + _parse_env_bool, +) +from tests.test_litellm_rust.support.callback_recorder import drain_logging +from tests.test_litellm_rust.support.recording_server import RecordingServer, recording_service + +CALLBACK_ATTRIBUTES: Final = ( + "callbacks", + "input_callback", + "success_callback", + "failure_callback", + "_async_input_callback", + "_async_success_callback", + "_async_failure_callback", +) +EXPECTED_FAILURE_REASONS: Final = { + "ocr/test_callbacks.py": "requires the OCR callback lifecycle implementation from #40070", + "ocr/test_guardrails.py": "requires the OCR guardrail lifecycle implementation from #40070", + "ocr/test_requests.py": "requires the OCR request and Azure authentication implementation from #40070", +} -def pytest_collection_modifyitems(items): - rust_enabled = os.environ.get("LITELLM_RUST", "").strip().lower() in { - "1", - "true", - "yes", - "on", - } - if not rust_enabled: - skip = pytest.mark.skip(reason="requires LITELLM_RUST=1 and a compiled Rust extension") +def _list_attribute(container: ModuleType, attribute: str) -> list[object]: + value: Final = getattr(container, attribute) + if not isinstance(value, list): + raise AssertionError(f"{container.__name__}.{attribute} is not a list") + return cast(list[object], value) + + +@contextmanager +def _isolated_list(container: ModuleType, attribute: str) -> Iterator[None]: + source: Final = _list_attribute(container, attribute) + original: Final = list(source) + source.clear() # mutable-ok: test isolation mutates global registries by design + try: + yield + finally: + source.clear() + source.extend(original) + setattr(container, attribute, source) + + +@contextmanager +def _rebound(container: object, attribute: str, value: object) -> Iterator[None]: + original: Final[object] = getattr(container, attribute) + setattr(container, attribute, value) + try: + yield + finally: + setattr(container, attribute, original) + + +@pytest_asyncio.fixture(autouse=True, loop_scope="function") +async def isolate_ocr_test_state() -> AsyncIterator[None]: + with ExitStack() as stack: + for attribute in CALLBACK_ATTRIBUTES: + stack.enter_context(_isolated_list(litellm, attribute)) + stack.enter_context(_isolated_list(litellm_logging, "_in_memory_loggers")) # pyright: ignore[reportPrivateUsage] # no public callback-cache accessor + stack.enter_context(_rebound(utils, "callback_list", [])) # rebind-ok: isolate legacy callback registry + stack.enter_context(_rebound(litellm, "cache", None)) # test-quality-ok: isolate process-global cache + stack.enter_context(_rebound(_CONFIGURATION, "override", None)) + executor: Final = ThreadPoolExecutor(thread_name_prefix="rust-ocr-test-logging") + stack.enter_context(_rebound(utils, "executor", executor)) + try: + yield + finally: + try: + await drain_logging() + finally: + await asyncio.to_thread(executor.shutdown, wait=True) + await GLOBAL_LOGGING_WORKER.stop() + + +@pytest.fixture +def recording_server() -> Generator[RecordingServer]: + with recording_service() as server: + yield server + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + for item in items: + if "test_litellm_rust" not in item.path.parts: + continue + relative_path: Final = "/".join(item.path.parts[item.path.parts.index("test_litellm_rust") + 1 :]) + reason: Final = EXPECTED_FAILURE_REASONS.get(relative_path) + if reason is not None: + item.add_marker(pytest.mark.xfail(reason=reason, strict=False)) + + if not _parse_env_bool(os.environ.get("LITELLM_RUST")): + skip: Final = pytest.mark.skip(reason="requires LITELLM_RUST=1 and a compiled Rust extension") for item in items: - item.add_marker(skip) + if "test_litellm_rust" in item.path.parts: + item.add_marker(skip) return try: from litellm.rust_bridge import _native # noqa: F401 # validates the installed extension except ImportError as error: - raise pytest.UsageError( - "LITELLM_RUST=1 requires a compiled litellm.rust_bridge._native extension" - ) from error + raise pytest.UsageError("LITELLM_RUST=1 requires a compiled litellm.rust_bridge._native extension") from error + + +@pytest.fixture +def isolated_azure_auth(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + "AZURE_AI_API_KEY", + "AZURE_AI_API_BASE", + "AZURE_AD_TOKEN", + "AZURE_TENANT_ID", + "AZURE_CLIENT_ID", + "AZURE_CLIENT_SECRET", + "AZURE_USERNAME", + "AZURE_PASSWORD", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "enable_azure_ad_token_refresh", False) diff --git a/tests/test_litellm_rust/ocr/__init__.py b/tests/test_litellm_rust/ocr/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/test_litellm_rust/ocr/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/test_litellm_rust/ocr/test_callbacks.py b/tests/test_litellm_rust/ocr/test_callbacks.py new file mode 100644 index 00000000000..b08446412c0 --- /dev/null +++ b/tests/test_litellm_rust/ocr/test_callbacks.py @@ -0,0 +1,513 @@ +import asyncio +import copy +import queue +import threading +from typing import Final + +import pytest + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.base_llm.ocr.transformation import OCRResponse +from tests.test_litellm_rust.support.callback_recorder import RecordingLogger +from tests.test_litellm_rust.support.requests import ( + OCR_DOCUMENT, + OCR_RESPONSE, + call_native_aocr, + call_native_ocr, + request_body, + request_headers, +) +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec + +pytestmark = pytest.mark.requires_rust_extension + + +@pytest.fixture +def ocr_server(recording_server: RecordingServer) -> RecordingServer: + recording_server.default_response = ResponseSpec(body=OCR_RESPONSE) + return recording_server + + +def call_native_ocr_with_callbacks(server: RecordingServer, callbacks: list[CustomLogger], **kwargs: object): + return call_native_ocr(server, callbacks=callbacks, **kwargs) + + +async def call_native_aocr_with_callbacks(server: RecordingServer, callbacks: list[CustomLogger], **kwargs: object): + return await call_native_aocr(server, callbacks=callbacks, **kwargs) + + +def test_native_ocr_pre_call_callback_receives_transformed_provider_request(ocr_server: RecordingServer) -> None: + observations: Final = [] + + class Observe(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + observations.append((model, copy.deepcopy(kwargs["additional_args"]))) + + call_native_ocr_with_callbacks(ocr_server, [Observe()], pages=[0]) + + assert len(observations) == 1 + model, additional_args = observations[0] + assert model == "mistral-ocr-latest" + assert additional_args["api_base"] == f"{ocr_server.base_url}/v1/ocr" + assert additional_args["complete_input_dict"] == { + "model": "mistral-ocr-latest", + "document": OCR_DOCUMENT, + "pages": [0], + } + + +@pytest.mark.parametrize("raise_after_edit", [False, True], ids=["callback-returns", "callback-raises"]) +def test_native_ocr_pre_call_body_edit_reaches_next_callback_and_provider( + ocr_server: RecordingServer, raise_after_edit: bool +) -> None: + observed: Final = [] + + class Edit(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + request_body(kwargs)["include_image_base64"] = True + if raise_after_edit: + raise RuntimeError("pre-call callback failed") + + class Observe(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + observed.append(copy.deepcopy(request_body(kwargs))) + + call_native_ocr_with_callbacks(ocr_server, [Edit(), Observe()], include_image_base64=False) + + assert observed[0]["include_image_base64"] is True + assert ocr_server.requests[0].body["include_image_base64"] is True + + +def test_native_ocr_pre_call_header_edit_reaches_next_callback_and_provider(ocr_server: RecordingServer) -> None: + observed: Final = [] + + class Edit(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + request_headers(kwargs)["x-audit-tag"] = "reviewed" + + class Observe(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + observed.append(dict(request_headers(kwargs))) + + call_native_ocr_with_callbacks(ocr_server, [Edit(), Observe()]) + + assert observed[0]["x-audit-tag"] == "reviewed" + assert ocr_server.requests[0].headers["x-audit-tag"] == "reviewed" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_ocr_pre_call_nested_document_edit_updates_caller_callback_and_provider_references( + ocr_server: RecordingServer, asynchronous: bool +) -> None: + original: Final = dict(OCR_DOCUMENT) + replacement_url: Final = "data:application/pdf;base64,ZGVm" + retained: Final = [] + aliases: Final = [] + + class Retain(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + aliases.append(request_body(kwargs)["document"] is original) + retained.append(request_body(kwargs)["document"]) + + class Edit(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + original["document_url"] = replacement_url + + arguments: Final = { + "model": "mistral/mistral-ocr-latest", + "document": original, + "api_key": "test-key", + "api_base": ocr_server.base_url, + "callbacks": [Retain(), Edit()], + } + response: Final = ( + await call_native_aocr(ocr_server, **arguments) + if asynchronous + else call_native_ocr(ocr_server, **arguments) + ) + + assert aliases == [True] + assert retained[0]["document_url"] == replacement_url + assert original["document_url"] == replacement_url + assert ocr_server.requests[0].body["document"]["document_url"] == replacement_url + assert response.pages[0].markdown == "native OCR response" + + +def test_native_ocr_pre_call_document_replacement_does_not_mutate_original_document( + ocr_server: RecordingServer, +) -> None: + original: Final = dict(OCR_DOCUMENT) + replacement: Final = {"type": "document_url", "document_url": "data:application/pdf;base64,ZGVm"} + retained: Final = [] + + class RetainAndReplace(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + body = request_body(kwargs) + retained.append(body["document"]) + body["document"] = replacement + + call_native_ocr( + ocr_server, + document=original, + callbacks=[RetainAndReplace()], + ) + + assert retained[0] is original + assert original["document_url"] == OCR_DOCUMENT["document_url"] + assert ocr_server.requests[0].body["document"] == replacement + + +def test_native_ocr_pre_call_body_rebinding_is_visible_to_callbacks_but_not_provider( + ocr_server: RecordingServer, +) -> None: + observed: Final = [] + + class Rebind(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + kwargs["additional_args"]["complete_input_dict"] = {"replacement": True} + + class Observe(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + observed.append(request_body(kwargs)) + + call_native_ocr_with_callbacks(ocr_server, [Rebind(), Observe()]) + + assert observed == [{"replacement": True}] + assert ocr_server.requests[0].body == {"model": "mistral-ocr-latest", "document": OCR_DOCUMENT} + + +def test_native_ocr_callback_retained_body_observes_later_callback_mutation(ocr_server: RecordingServer) -> None: + queued: Final = [] + + class QueuePayload(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + queued.append(request_body(kwargs)) + + class Edit(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + request_body(kwargs)["queued-edit"] = True + + call_native_ocr_with_callbacks(ocr_server, [QueuePayload(), Edit()]) + + assert queued[0]["queued-edit"] is True + + +def test_native_ocr_success_callback_receives_state_added_by_pre_call_callback(ocr_server: RecordingServer) -> None: + token: Final = object() + terminal_tokens: queue.SimpleQueue[object] = queue.SimpleQueue() + finished: Final = threading.Event() + + class Stash(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + kwargs["test-token"] = token + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + terminal_tokens.put(kwargs["test-token"]) + finished.set() + + call_native_ocr_with_callbacks(ocr_server, [Stash()]) + + assert finished.wait(10) + assert terminal_tokens.get_nowait() is token + + +@pytest.mark.asyncio +async def test_native_aocr_success_callback_receives_call_id_metadata_and_response( + ocr_server: RecordingServer, +) -> None: + recorder: Final = RecordingLogger() + + await call_native_aocr_with_callbacks( + ocr_server, + [recorder], + litellm_call_id="ocr-success", + metadata={"source": "callback-test"}, + ) + events: Final = await recorder.wait_for_async("async_log_success_event") + + assert len(events) == 1 + assert events[0].call_type == "aocr" + assert events[0].kwargs["litellm_call_id"] == "ocr-success" + assert events[0].kwargs["litellm_params"]["metadata"]["source"] == "callback-test" + assert events[0].response.pages[0].markdown == "native OCR response" + + +@pytest.mark.asyncio +async def test_native_aocr_failure_callbacks_receive_call_type_error_and_no_response( + ocr_server: RecordingServer, +) -> None: + ocr_server.enqueue(ResponseSpec(body={"message": "provider unavailable"}, status=500)) + observations: Final = [] + + class Observe(CustomLogger): + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + observations.append(("sync", kwargs["call_type"], kwargs["exception"], response_obj)) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + observations.append(("async", kwargs["call_type"], kwargs["exception"], response_obj)) + + with pytest.raises(litellm.InternalServerError): + await call_native_aocr_with_callbacks(ocr_server, [Observe()]) + + assert [observation[0] for observation in observations] == ["sync", "async"] + assert all(observation[1] == "aocr" for observation in observations) + assert all(isinstance(observation[2], litellm.InternalServerError) for observation in observations) + assert all(observation[3] is None for observation in observations) + + +@pytest.mark.asyncio +async def test_native_aocr_pre_call_callback_runs_on_caller_loop_and_thread(ocr_server: RecordingServer) -> None: + caller_loop: Final = asyncio.get_running_loop() + caller_thread: Final = threading.current_thread() + recorder: Final = RecordingLogger() + + await call_native_aocr_with_callbacks(ocr_server, [recorder]) + + events: Final = await recorder.wait_for_async("log_pre_api_call") + assert len(events) == 1 + assert events[0].loop is caller_loop + assert events[0].thread is caller_thread + + +@pytest.mark.asyncio +async def test_native_aocr_failure_callbacks_receive_state_added_by_pre_call_callback( + ocr_server: RecordingServer, +) -> None: + ocr_server.enqueue(ResponseSpec(body={"message": "provider unavailable"}, status=500)) + token: Final = object() + observed: Final = [] + + class TrackInFlightRequest(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + kwargs["request-token"] = token + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + observed.append(("sync", kwargs["request-token"])) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + observed.append(("async", kwargs["request-token"])) + + with pytest.raises(litellm.InternalServerError): + await call_native_aocr_with_callbacks(ocr_server, [TrackInFlightRequest()]) + + assert [event for event, _ in observed] == ["sync", "async"] + assert all(observed_token is token for _, observed_token in observed) + + +@pytest.mark.asyncio +async def test_native_aocr_callback_error_does_not_mask_provider_error_or_skip_later_failure_callbacks( + ocr_server: RecordingServer, +) -> None: + ocr_server.enqueue(ResponseSpec(body={"message": "provider unavailable"}, status=500)) + recorder: Final = RecordingLogger() + + class FailingCallback(CustomLogger): + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + raise RuntimeError("failure callback failed") + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + raise RuntimeError("failure callback failed") + + with pytest.raises(litellm.InternalServerError) as caught: + await call_native_aocr_with_callbacks(ocr_server, [FailingCallback(), recorder]) + + sync_events: Final = tuple(event for event in recorder.events if event.name == "log_failure_event") + async_events: Final = tuple(event for event in recorder.events if event.name == "async_log_failure_event") + assert len(sync_events) == 1 + assert len(async_events) == 1 + assert sync_events[0].kwargs["exception"] is caught.value + assert async_events[0].kwargs["exception"] is caught.value + assert "async_log_success_event" not in recorder.names + + +def test_native_ocr_dispatches_each_callback_phase_once_when_logger_is_registered_multiple_times( + ocr_server: RecordingServer, +) -> None: + recorder: Final = RecordingLogger() + + call_native_ocr_with_callbacks( + ocr_server, + [recorder, recorder], + success_callback=[recorder], + failure_callback=[recorder], + ) + recorder.wait_for("log_success_event") + + assert recorder.names.count("log_pre_api_call") == 1 + assert recorder.names.count("logging_hook") == 1 + assert recorder.names.count("log_success_event") == 1 + assert "log_failure_event" not in recorder.names + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_azure_ocr_resolves_token_before_pre_call_on_caller_context( + ocr_server: RecordingServer, + isolated_azure_auth: None, + asynchronous: bool, +) -> None: + from contextvars import ContextVar + context: Final = ContextVar("azure-token-context", default="missing") + context.set("caller") + caller_thread: Final = threading.current_thread() + caller_loop: Final = asyncio.get_running_loop() + observations: Final = [] + + class Provider: + def __call__(self) -> str: + assert context.get() == "caller" + assert threading.current_thread() is caller_thread + assert asyncio.get_running_loop() is caller_loop + observations.append("token") + return "caller-token" + + class Edit(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + assert request_headers(kwargs)["Authorization"] == "Bearer caller-token" + observations.append("pre_call") + request_headers(kwargs)["Authorization"] = "Bearer edited" + + provider: Final = Provider() + arguments: Final = { + "model": "azure_ai/mistral-ocr-latest", + "api_key": None, + "azure_ad_token_provider": provider, + "callbacks": [Edit()], + } + response: Final = ( + await call_native_aocr(ocr_server, **arguments) + if asynchronous + else call_native_ocr(ocr_server, **arguments) + ) + assert response.pages[0].markdown == "native OCR response" + assert observations == ["token", "pre_call"] + assert ocr_server.requests[0].headers["authorization"] == "Bearer edited" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_azure_ocr_token_provider_can_make_nested_native_ocr_call( + ocr_server: RecordingServer, + isolated_azure_auth: None, + asynchronous: bool, +) -> None: + ocr_server.expected_requests = 2 + calls: Final = [] + + def provider() -> str: + calls.append("token") + nested: Final = call_native_ocr(ocr_server) + assert nested.pages[0].markdown == "native OCR response" + return "outer-token" + + arguments: Final = { + "model": "azure_ai/mistral-ocr-latest", + "api_key": None, + "azure_ad_token_provider": provider, + } + response: Final = ( + await call_native_aocr(ocr_server, **arguments) + if asynchronous + else call_native_ocr(ocr_server, **arguments) + ) + assert response.pages[0].markdown == "native OCR response" + assert calls == ["token"] + assert [request.headers["authorization"] for request in ocr_server.requests] == [ + "Bearer test-key", + "Bearer outer-token", + ] + + +@pytest.mark.asyncio +async def test_concurrent_native_azure_ocr_calls_isolate_token_results_and_error( + ocr_server: RecordingServer, + isolated_azure_auth: None, +) -> None: + ocr_server.expected_requests = 2 + + async def request(token: str, fail: bool) -> object: + def provider() -> str: + if fail: + raise ValueError(token) + return token + + return await call_native_aocr( + ocr_server, + model="azure_ai/mistral-ocr-latest", + api_key=None, + azure_ad_token_provider=provider, + ) + + responses: Final = await asyncio.gather( + request("first", False), + request("failed", True), + request("second", False), + return_exceptions=True, + ) + assert isinstance(responses[0], OCRResponse) + assert isinstance(responses[1], litellm.APIConnectionError) + assert "Failed to get Azure AD token: failed" in str(responses[1]) + assert isinstance(responses[2], OCRResponse) + assert sorted(request.headers["authorization"] for request in ocr_server.requests) == [ + "Bearer first", + "Bearer second", + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("outcome", ["success", "failure", "cancellation"]) +async def test_native_azure_ocr_releases_token_provider_after_terminal_outcome( + ocr_server: RecordingServer, + isolated_azure_auth: None, + outcome: str, +) -> None: + import gc + import weakref + from tests.test_litellm_rust.support.callback_recorder import drain_logging + class Provider: + def __call__(self) -> str: + if outcome == "failure": + raise ValueError("unavailable") + return "caller-token" + + async def invoke() -> weakref.ReferenceType[Provider]: + provider: Final = Provider() + reference: Final = weakref.ref(provider) + if outcome == "failure": + ocr_server.expected_requests = 0 + with pytest.raises(litellm.APIConnectionError): + await call_native_aocr( + ocr_server, model="azure_ai/mistral-ocr-latest", api_key=None, azure_ad_token_provider=provider + ) + elif outcome == "cancellation": + ocr_server.enqueue(ResponseSpec(body=OCR_RESPONSE, delay=0.1)) + task: Final = asyncio.create_task( + call_native_aocr( + ocr_server, + model="azure_ai/mistral-ocr-latest", + api_key=None, + azure_ad_token_provider=provider, + ) + ) + await ocr_server.wait_for_requests(1) + assert reference() is provider + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + response: Final = await call_native_aocr( + ocr_server, + model="azure_ai/mistral-ocr-latest", + api_key=None, + azure_ad_token_provider=provider, + ) + assert response.pages[0].markdown == "native OCR response" + return reference + + reference: Final = await invoke() + await drain_logging() + await asyncio.sleep(0) + gc.collect() + assert reference() is None diff --git a/tests/test_litellm_rust/ocr/test_dispatch.py b/tests/test_litellm_rust/ocr/test_dispatch.py new file mode 100644 index 00000000000..a6c76bc5d0e --- /dev/null +++ b/tests/test_litellm_rust/ocr/test_dispatch.py @@ -0,0 +1,44 @@ +from typing import Final +from unittest.mock import Mock + +import pytest + +import litellm +from litellm.llms.base_llm.ocr.transformation import OCRResponse +from litellm.ocr import main as ocr_main +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec +from tests.test_litellm_rust.support.requests import OCR_DOCUMENT, OCR_MODEL, OCR_RESPONSE + +pytestmark = pytest.mark.requires_rust_extension + + +@pytest.fixture +def ocr_server(recording_server: RecordingServer) -> RecordingServer: + recording_server.default_response = ResponseSpec(body=OCR_RESPONSE) + return recording_server + + +@pytest.mark.parametrize("rust_enabled", [True, False], ids=["enabled", "disabled"]) +def test_public_ocr_dispatches_according_to_rust_setting( + ocr_server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + rust_enabled: bool, +) -> None: + rust_call: Final = Mock(wraps=ocr_main.rust_ocr_bridge.ocr) + python_call: Final = Mock(wraps=ocr_main.base_llm_http_handler.ocr) + monkeypatch.setattr(ocr_main.rust_ocr_bridge, "ocr", rust_call) + monkeypatch.setattr(ocr_main.base_llm_http_handler, "ocr", python_call) + litellm.rust(rust_enabled) + + response: Final = litellm.ocr( + model=OCR_MODEL, + document=OCR_DOCUMENT, + api_key="test-key", + api_base=ocr_server.base_url, + ) + + assert isinstance(response, OCRResponse) + assert response.pages[0].markdown == "native OCR response" + assert rust_call.call_count == int(rust_enabled) + assert python_call.call_count == int(not rust_enabled) + assert len(ocr_server.requests) == 1 diff --git a/tests/test_litellm_rust/ocr/test_guardrails.py b/tests/test_litellm_rust/ocr/test_guardrails.py new file mode 100644 index 00000000000..f6fc1c7cb8d --- /dev/null +++ b/tests/test_litellm_rust/ocr/test_guardrails.py @@ -0,0 +1,74 @@ +from typing import Final + +import pytest +from fastapi import HTTPException + +import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail +from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks +from litellm.types.utils import CallTypes +from tests.test_litellm_rust.support.callback_recorder import RecordingLogger +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec +from tests.test_litellm_rust.support.requests import OCR_RESPONSE, call_native_aocr + +pytestmark = pytest.mark.requires_rust_extension + + +@pytest.fixture +def ocr_server(recording_server: RecordingServer) -> RecordingServer: + recording_server.default_response = ResponseSpec(body=OCR_RESPONSE) + return recording_server + + +class ReplaceOCRMarkdown(CustomGuardrail): + def __init__(self) -> None: + super().__init__( + guardrail_name="replace-ocr-markdown", event_hook=GuardrailEventHooks.post_call, default_on=True + ) + self.call_types: list[CallTypes] = [] + + async def async_post_call_success_deployment_hook(self, request_data, response, call_type): + self.call_types.append(call_type) + reviewed_page: Final = response.pages[0].model_copy(update={"markdown": "Reviewed OCR"}) + return response.model_copy(update={"pages": [reviewed_page]}) + + +@pytest.mark.asyncio +async def test_native_aocr_post_call_content_filter_blocks_matching_markdown( + ocr_server: RecordingServer, +) -> None: + guardrail: Final = ContentFilterGuardrail( + guardrail_name="block-native-ocr-markdown", + event_hook=GuardrailEventHooks.post_call, + blocked_words=[BlockedWord(keyword="native OCR response", action=ContentFilterAction.BLOCK)], + ) + litellm.callbacks.append(guardrail) + + with pytest.raises(HTTPException, match="Content blocked") as blocked: + await call_native_aocr(ocr_server, guardrails=[guardrail.guardrail_name]) + + assert blocked.value.status_code == 400 + assert len(ocr_server.requests) == 1 + + +@pytest.mark.asyncio +async def test_native_aocr_post_call_replacement_reaches_caller_and_success_callback( + ocr_server: RecordingServer, +) -> None: + guardrail: Final = ReplaceOCRMarkdown() + recorder: Final = RecordingLogger() + litellm.callbacks.append(guardrail) + + response: Final = await call_native_aocr( + ocr_server, + callbacks=[recorder], + guardrails=[guardrail.guardrail_name], + ) + success_events: Final = await recorder.wait_for_async("async_log_success_event") + + assert guardrail.call_types == [CallTypes.aocr] + assert response.pages[0].markdown == "Reviewed OCR" + assert len(success_events) == 1 + assert success_events[0].response.pages[0].markdown == "Reviewed OCR" + assert "guardrails" not in ocr_server.requests[0].body diff --git a/tests/test_litellm_rust/ocr/test_requests.py b/tests/test_litellm_rust/ocr/test_requests.py new file mode 100644 index 00000000000..d241fe08fc8 --- /dev/null +++ b/tests/test_litellm_rust/ocr/test_requests.py @@ -0,0 +1,434 @@ +from typing import Final + +import pytest + +import litellm +from litellm.llms.base_llm.ocr.transformation import OCRResponse +from tests.test_litellm_rust.support.callback_recorder import RecordingLogger +from tests.test_litellm_rust.support.requests import ( + OCR_DOCUMENT, + OCR_RESPONSE, + call_native_aocr, + call_native_ocr, +) +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec + +pytestmark = pytest.mark.requires_rust_extension + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_azure_ocr_uses_token_provider_result_as_bearer_token( + ocr_server: RecordingServer, isolated_azure_auth: None, asynchronous: bool +) -> None: + calls: Final = [] + + def token_provider() -> str: + calls.append("token") + return "callback-token" + + arguments: Final = { + "model": "azure_ai/mistral-ocr-latest", + "api_key": None, + "azure_ad_token_provider": token_provider, + } + response: Final = ( + await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments) + ) + + assert calls == ["token"] + assert response.pages[0].markdown == "native OCR response" + assert_native_request(ocr_server) + assert ocr_server.requests[0].headers["authorization"] == "Bearer callback-token" + + +@pytest.fixture +def ocr_server(recording_server: RecordingServer) -> RecordingServer: + recording_server.default_response = ResponseSpec(body=OCR_RESPONSE) + return recording_server + + +def assert_native_request(server: RecordingServer) -> None: + assert len(server.requests) == 1 + assert not server.requests[0].headers.get("user-agent", "").startswith("python-httpx") + + +def test_native_ocr_sends_model_and_document_to_mistral_ocr_path(ocr_server: RecordingServer) -> None: + response: Final = call_native_ocr(ocr_server) + + assert response.pages[0].markdown == "native OCR response" + assert_native_request(ocr_server) + assert ocr_server.requests[0].path == "/v1/ocr" + assert ocr_server.requests[0].body == {"model": "mistral-ocr-latest", "document": OCR_DOCUMENT} + + +def test_native_ocr_prepares_file_document_like_python(ocr_server: RecordingServer) -> None: + response: Final = call_native_ocr( + ocr_server, + document={"type": "file", "file": b"%PDF-1.4", "mime_type": "application/pdf"}, + ) + + assert response.pages[0].markdown == "native OCR response" + assert_native_request(ocr_server) + assert ocr_server.requests[0].body == { + "model": "mistral-ocr-latest", + "document": { + "type": "document_url", + "document_url": "data:application/pdf;base64,JVBERi0xLjQ=", + }, + } + + +def test_native_ocr_sends_pages_and_image_options(ocr_server: RecordingServer) -> None: + call_native_ocr(ocr_server, pages=[0, 2], include_image_base64=True) + + assert ocr_server.requests[0].body["pages"] == [0, 2] + assert ocr_server.requests[0].body["include_image_base64"] is True + + +def test_native_ocr_merges_custom_headers_with_authorization(ocr_server: RecordingServer) -> None: + call_native_ocr(ocr_server, extra_headers={"x-trace-id": "trace-1"}) + + assert ocr_server.requests[0].headers["authorization"] == "Bearer test-key" + assert ocr_server.requests[0].headers["x-trace-id"] == "trace-1" + + +def test_native_mistral_ocr_uses_environment_api_key_when_argument_is_missing( + ocr_server: RecordingServer, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("MISTRAL_API_KEY", "environment-key") + + call_native_ocr(ocr_server, api_key=None) + + assert ocr_server.requests[0].headers["authorization"] == "Bearer environment-key" + + +def test_native_mistral_ocr_prefers_explicit_api_key_over_environment( + ocr_server: RecordingServer, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("MISTRAL_API_KEY", "environment-key") + + call_native_ocr(ocr_server) + + assert ocr_server.requests[0].headers["authorization"] == "Bearer test-key" + + +def test_native_azure_ocr_uses_environment_endpoint_and_api_key( + ocr_server: RecordingServer, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("AZURE_AI_API_KEY", "azure-key") + monkeypatch.setenv("AZURE_AI_API_BASE", ocr_server.base_url) + + call_native_ocr(ocr_server, model="azure_ai/pixtral-12b-2409", api_key=None, api_base=None) + + assert_native_request(ocr_server) + assert ocr_server.requests[0].path == "/providers/mistral/azure/ocr" + assert ocr_server.requests[0].headers["authorization"] == "Bearer azure-key" + + +def test_native_vertex_ocr_builds_path_from_project_and_location(ocr_server: RecordingServer) -> None: + call_native_ocr( + ocr_server, + model="vertex_ai/mistral-ocr-2505", + api_key="vertex-token", + vertex_project="project-1", + vertex_location="us-central1", + ) + + assert_native_request(ocr_server) + assert ocr_server.requests[0].path == ( + "/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-2505:rawPredict" + ) + + +def test_native_ocr_normalizes_provider_response_model_and_usage(ocr_server: RecordingServer) -> None: + response: Final = call_native_ocr(ocr_server) + + assert isinstance(response, OCRResponse) + assert response.model == "mistral-ocr-latest" + assert response.usage_info.pages_processed == 1 + + +def test_native_ocr_maps_provider_400_without_exposing_response_body(ocr_server: RecordingServer) -> None: + ocr_server.enqueue(ResponseSpec(body={"message": "invalid OCR request"}, status=400)) + + with pytest.raises(litellm.BadRequestError) as caught: + call_native_ocr(ocr_server) + + assert caught.value.status_code == 400 + assert caught.value.model == "mistral-ocr-latest" + assert caught.value.llm_provider == "mistral" + assert "invalid OCR request" not in str(caught.value) + + +def test_native_ocr_raises_transport_error_when_request_exceeds_timeout(ocr_server: RecordingServer) -> None: + ocr_server.enqueue(ResponseSpec(body=OCR_RESPONSE, delay=0.2)) + + with pytest.raises(RuntimeError, match="OCR transport failed"): + call_native_ocr(ocr_server, timeout=0.01) + + assert len(ocr_server.requests) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize( + "credentials, expected_token, expected_calls", + [ + ({"api_key": "resource-key"}, "resource-key", 0), + ({"azure_ad_token": "static-token"}, "callback-1", 1), + ({"extra_headers": {"Authorization": "Bearer override"}}, "override", 1), + ], + ids=["api-key-skips-provider", "provider-overrides-static-token", "header-overrides-provider"], +) +async def test_native_azure_ocr_applies_python_credential_precedence( + ocr_server: RecordingServer, + isolated_azure_auth: None, + asynchronous: bool, + credentials: dict[str, object], + expected_token: str, + expected_calls: int, +) -> None: + calls: Final = [] + + def token_provider() -> str: + calls.append("token") + return f"callback-{len(calls)}" + + arguments: Final = { + "model": "azure_ai/mistral-ocr-latest", + "api_key": None, + "azure_ad_token_provider": token_provider, + **credentials, + } + response: Final = ( + await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments) + ) + assert response.pages[0].markdown == "native OCR response" + assert len(calls) == expected_calls + assert len(ocr_server.requests) == 1 + assert ocr_server.requests[0].headers["authorization"] == f"Bearer {expected_token}" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_azure_ocr_calls_token_provider_for_each_request( + ocr_server: RecordingServer, + isolated_azure_auth: None, + asynchronous: bool, +) -> None: + calls: Final = [] + ocr_server.expected_requests = 2 + + def token_provider() -> str: + calls.append("token") + return f"callback-{len(calls)}" + + for _ in range(2): + arguments: Final = { + "model": "azure_ai/mistral-ocr-latest", + "api_key": None, + "azure_ad_token_provider": token_provider, + } + response: Final = ( + await call_native_aocr(ocr_server, **arguments) + if asynchronous + else call_native_ocr(ocr_server, **arguments) + ) + assert response.pages[0].markdown == "native OCR response" + assert len(calls) == 2 + assert [request.headers["authorization"] for request in ocr_server.requests] == [ + "Bearer callback-1", + "Bearer callback-2", + ] + + +class TokenAbort(BaseException): + pass + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize( + "failure", + ["non_string", "type_error", "ordinary", "abort"], + ids=["non-string-result", "type-error", "value-error", "base-exception"], +) +async def test_native_azure_ocr_token_provider_failure_prevents_pre_call_callback_and_request( + ocr_server: RecordingServer, + isolated_azure_auth: None, + asynchronous: bool, + failure: str, +) -> None: + ocr_server.expected_requests = 0 + calls: Final = [] + recorder: Final = RecordingLogger() + original: Final = { + "type_error": TypeError("token type"), + "ordinary": ValueError("token unavailable"), + "abort": TokenAbort("abort"), + } + + def token_provider() -> object: + calls.append("token") + if failure == "non_string": + return 123 + raise original[failure] + + arguments: Final = { + "model": "azure_ai/mistral-ocr-latest", + "api_key": None, + "azure_ad_token_provider": token_provider, + "callbacks": [recorder], + } + expected: Final = TokenAbort if failure == "abort" else litellm.APIConnectionError + with pytest.raises(expected) as caught: + await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments) + assert calls == ["token"] + assert ocr_server.requests == [] + assert "log_pre_api_call" not in recorder.names + if failure == "ordinary": + assert "Failed to get Azure AD token: token unavailable" in str(caught.value) + assert isinstance(caught.value.__context__, RuntimeError) + assert caught.value.__context__.__cause__ is original[failure] + elif failure == "abort": + assert caught.value is original[failure] + elif failure == "type_error": + assert caught.value.__context__ is original[failure] + else: + assert isinstance(caught.value.__context__, TypeError) + + +@pytest.mark.parametrize( + "configuration", + [ + {"azure_ad_token": "oidc/assertion", "client_id": "client", "tenant_id": "tenant"}, + {"model": "azure_ai/doc-intelligence/prebuilt-read"}, + ], + ids=["oidc-assertion", "document-intelligence-model"], +) +def test_native_azure_ocr_rejects_unsupported_configuration_before_token_or_callbacks( + ocr_server: RecordingServer, + isolated_azure_auth: None, + configuration: dict[str, object], +) -> None: + ocr_server.expected_requests = 0 + calls: Final = [] + recorder: Final = RecordingLogger() + + def provider() -> str: + calls.append("token") + return "unused" + + arguments: Final = { + "model": "azure_ai/mistral-ocr-latest", + "api_key": None, + "azure_ad_token_provider": provider, + "callbacks": [recorder], + **configuration, + } + with pytest.raises(NotImplementedError): + call_native_ocr(ocr_server, **arguments) + assert calls == [] + assert recorder.events == () + assert ocr_server.requests == [] + + +@pytest.mark.asyncio +async def test_native_azure_ocr_validates_endpoint_before_calling_token_provider( + ocr_server: RecordingServer, + isolated_azure_auth: None, +) -> None: + ocr_server.expected_requests = 0 + calls: Final = [] + + def provider() -> str: + calls.append("token") + return "unused" + + with pytest.raises(litellm.APIConnectionError, match="Missing Azure AI API Base"): + await call_native_aocr( + ocr_server, + model="azure_ai/mistral-ocr-latest", + api_key=None, + api_base=None, + azure_ad_token_provider=provider, + ) + assert calls == [] + assert ocr_server.requests == [] + + +@pytest.mark.asyncio +async def test_native_azure_ocr_does_not_fall_back_to_static_token_after_empty_provider_result( + ocr_server: RecordingServer, + isolated_azure_auth: None, +) -> None: + ocr_server.expected_requests = 0 + + def provider() -> str: + return "" + + with pytest.raises(litellm.APIConnectionError, match="Missing Azure AI credentials"): + await call_native_aocr( + ocr_server, + model="azure_ai/mistral-ocr-latest", + api_key=None, + azure_ad_token="static-token", + azure_ad_token_provider=provider, + ) + assert ocr_server.requests == [] + + +@pytest.mark.asyncio +async def test_native_azure_ocr_ignores_falsey_token_provider_and_uses_static_token( + ocr_server: RecordingServer, + isolated_azure_auth: None, +) -> None: + calls: Final = [] + + class Provider: + def __bool__(self) -> bool: + return False + + def __call__(self) -> str: + calls.append("token") + return "unused" + + response: Final = await call_native_aocr( + ocr_server, + model="azure_ai/mistral-ocr-latest", + api_key=None, + azure_ad_token="static-token", + azure_ad_token_provider=Provider(), + ) + assert response.pages[0].markdown == "native OCR response" + assert calls == [] + assert ocr_server.requests[0].headers["authorization"] == "Bearer static-token" + + +@pytest.mark.asyncio +async def test_native_azure_ocr_rejects_coroutine_returned_by_sync_token_provider( + ocr_server: RecordingServer, + isolated_azure_auth: None, +) -> None: + ocr_server.expected_requests = 0 + calls: Final = [] + + async def acquire() -> str: + calls.append("awaited") + return "unused" + + coroutine: Final = acquire() + + def provider() -> object: + return coroutine + + try: + with pytest.raises(litellm.APIConnectionError, match="Azure AD token must be a string"): + await call_native_aocr( + ocr_server, model="azure_ai/mistral-ocr-latest", api_key=None, azure_ad_token_provider=provider + ) + finally: + coroutine.close() + assert calls == [] + assert ocr_server.requests == [] diff --git a/tests/test_litellm_rust/support/__init__.py b/tests/test_litellm_rust/support/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/test_litellm_rust/support/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/test_litellm_rust/support/callback_recorder.py b/tests/test_litellm_rust/support/callback_recorder.py new file mode 100644 index 00000000000..6de011b1414 --- /dev/null +++ b/tests/test_litellm_rust/support/callback_recorder.py @@ -0,0 +1,101 @@ +import asyncio +import copy +import threading +import time +from dataclasses import dataclass +from typing import Final + +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker + + +async def drain_logging(worker: LoggingWorker = GLOBAL_LOGGING_WORKER) -> None: + await asyncio.sleep(0) + worker.start() + await asyncio.wait_for(worker.flush(), timeout=10) + + +@dataclass(frozen=True, slots=True) +class HookEvent: + name: str + call_type: str | None + thread: threading.Thread + loop: asyncio.AbstractEventLoop | None + kwargs: object + response: object + + +class RecordingLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self._events: list[HookEvent] = [] + self._condition = threading.Condition() + + @property + def events(self) -> tuple[HookEvent, ...]: + with self._condition: + return tuple(self._events) + + @property + def names(self) -> tuple[str, ...]: + return tuple(event.name for event in self.events) + + def _record(self, name: str, kwargs: object = None, response: object = None) -> None: + details: Final = kwargs if isinstance(kwargs, dict) else {} + try: + snapshot: Final = copy.deepcopy(details) + except Exception: + snapshot = dict(details) + if "exception" in details: + snapshot["exception"] = details["exception"] + try: + loop: Final = asyncio.get_running_loop() + except RuntimeError: + loop = None + event: Final = HookEvent( + name=name, + call_type=details.get("call_type"), + thread=threading.current_thread(), + loop=loop, + kwargs=snapshot, + response=response, + ) + with self._condition: + self._events.append(event) + self._condition.notify_all() + + def wait_for(self, name: str, count: int = 1, timeout: float = 10) -> tuple[HookEvent, ...]: + deadline: Final = time.monotonic() + timeout + with self._condition: + while sum(event.name == name for event in self._events) < count: + remaining: Final = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(f"Timed out waiting for {count} {name} events; saw {self.names}") + self._condition.wait(remaining) + return tuple(event for event in self._events if event.name == name) + + async def wait_for_async(self, name: str, count: int = 1, timeout: float = 10) -> tuple[HookEvent, ...]: + await asyncio.wait_for(asyncio.to_thread(self.wait_for, name, count, timeout), timeout=timeout + 1) + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=timeout) + return tuple(event for event in self.events if event.name == name) + + def log_pre_api_call(self, model, _messages, kwargs): + self._record("log_pre_api_call", kwargs) + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + self._record("log_success_event", kwargs, response_obj) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self._record("async_log_success_event", kwargs, response_obj) + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + self._record("log_failure_event", kwargs, response_obj) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self._record("async_log_failure_event", kwargs, response_obj) + + def logging_hook(self, kwargs, result, call_type): + self._record("logging_hook", kwargs, result) + return kwargs, result diff --git a/tests/test_litellm_rust/support/recording_server.py b/tests/test_litellm_rust/support/recording_server.py new file mode 100644 index 00000000000..5a9b9497c6e --- /dev/null +++ b/tests/test_litellm_rust/support/recording_server.py @@ -0,0 +1,108 @@ +import asyncio +import copy +import json +import threading +import time +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final + + +@dataclass +class RecordedRequest: + method: str + path: str + headers: dict[str, str] + raw_body: bytes + body: object | None + + +@dataclass +class ResponseSpec: + body: object + status: int = 200 + headers: dict[str, str] = field(default_factory=dict) + delay: float = 0 + + +@dataclass +class RecordingServer: + server: ThreadingHTTPServer + requests: list[RecordedRequest] + responses: list[ResponseSpec] + default_response: ResponseSpec + expected_requests: int | None = 1 + + @property + def base_url(self) -> str: + host, port = self.server.server_address + return f"http://{host}:{port}" + + def enqueue(self, response: ResponseSpec) -> None: + self.responses.append(response) + + async def wait_for_requests(self, count: int) -> None: + async with asyncio.timeout(2): + while len(self.requests) < count: + await asyncio.sleep(0.01) + + +@contextmanager +def recording_service() -> Iterator[RecordingServer]: + requests: list[RecordedRequest] = [] + responses: list[ResponseSpec] = [] + + class Handler(BaseHTTPRequestHandler): + def _handle(self) -> None: + content_length: Final = int(self.headers.get("Content-Length", "0")) + raw_body: Final = self.rfile.read(content_length) if content_length else b"" + body: Final = json.loads(raw_body) if raw_body else None + requests.append( + RecordedRequest( + method=self.command, + path=self.path, + headers={name.lower(): value for name, value in self.headers.items()}, + raw_body=raw_body, + body=body, + ) + ) + response: Final = responses.pop(0) if responses else copy.deepcopy(recording_server.default_response) + if response.delay: + time.sleep(response.delay) + payload: Final = json.dumps(response.body).encode() + self.send_response(response.status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + for name, value in response.headers.items(): + self.send_header(name, value) + self.end_headers() + try: + self.wfile.write(payload) + except (BrokenPipeError, ConnectionResetError): + pass + + do_POST = _handle + + def log_message(self, format: str, *args: object) -> None: + pass + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + thread.start() + try: + recording_server = RecordingServer( + server=server, + requests=requests, + responses=responses, + default_response=ResponseSpec(body={}), + ) + yield recording_server + finally: + server.shutdown() + server.server_close() + thread.join() + if recording_server.expected_requests is not None: + assert len(recording_server.requests) == recording_server.expected_requests + assert recording_server.responses == [] diff --git a/tests/test_litellm_rust/support/requests.py b/tests/test_litellm_rust/support/requests.py new file mode 100644 index 00000000000..d681d752ffd --- /dev/null +++ b/tests/test_litellm_rust/support/requests.py @@ -0,0 +1,59 @@ +from typing import Final + +import litellm +from litellm.llms.base_llm.ocr.transformation import OCRResponse +from litellm.rust_bridge import ocr as native_ocr +from tests.test_litellm_rust.support.recording_server import RecordingServer + +OCR_DOCUMENT: Final = {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"} +OCR_MODEL: Final = "mistral/mistral-ocr-latest" +OCR_RESPONSE: Final = { + "pages": [{"index": 0, "markdown": "native OCR response", "images": [], "dimensions": None}], + "model": "mistral-ocr-latest", + "usage_info": {"pages_processed": 1, "doc_size_bytes": 3}, +} + + +def ocr_arguments(server: RecordingServer, **kwargs: object) -> dict[str, object]: + return { + "model": OCR_MODEL, + "document": dict(OCR_DOCUMENT), + "api_key": "test-key", + "api_base": server.base_url, + **kwargs, + } + + +def call_ocr(server: RecordingServer, **kwargs: object) -> OCRResponse: + response: Final = litellm.ocr(**ocr_arguments(server, **kwargs)) + if not isinstance(response, OCRResponse): + raise TypeError(f"Expected OCRResponse, got {type(response).__name__}") + return response + + +async def call_aocr(server: RecordingServer, **kwargs: object) -> OCRResponse: + return await litellm.aocr(**ocr_arguments(server, **kwargs)) + + +def call_native_ocr(server: RecordingServer, **kwargs: object) -> OCRResponse: + return native_ocr.ocr(ocr_arguments(server, **kwargs)) + + +async def call_native_aocr(server: RecordingServer, **kwargs: object) -> OCRResponse: + return await native_ocr.aocr(ocr_arguments(server, **kwargs)) + + +def request_body(kwargs: dict[str, object]) -> dict[str, object]: + additional_args = kwargs["additional_args"] + assert isinstance(additional_args, dict) + body = additional_args["complete_input_dict"] + assert isinstance(body, dict) + return body + + +def request_headers(kwargs: dict[str, object]) -> dict[str, object]: + additional_args = kwargs["additional_args"] + assert isinstance(additional_args, dict) + headers = additional_args["headers"] + assert isinstance(headers, dict) + return headers diff --git a/tests/test_litellm_rust/test_ocr.py b/tests/test_litellm_rust/test_ocr.py index d5b1fce1139..1293de9ee0e 100644 --- a/tests/test_litellm_rust/test_ocr.py +++ b/tests/test_litellm_rust/test_ocr.py @@ -1,31 +1,34 @@ import json import threading +from collections.abc import Generator +from dataclasses import dataclass from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final import pytest -import litellm +from litellm.rust_bridge import ocr as rust_ocr_bridge pytestmark = pytest.mark.requires_rust_extension +@dataclass(frozen=True, slots=True) +class RecordedOCRRequest: + body: object + + @pytest.fixture -def ocr_server(): - requests = [] +def ocr_server() -> Generator[tuple[ThreadingHTTPServer, list[RecordedOCRRequest]]]: + requests: Final[list[RecordedOCRRequest]] = [] class Handler(BaseHTTPRequestHandler): - def do_POST(self): + def do_POST(self) -> None: requests.append( - { - "headers": {name.lower(): value for name, value in self.headers.items()}, - "body": json.loads(self.rfile.read(int(self.headers["Content-Length"]))), - } + RecordedOCRRequest( + body=json.loads(self.rfile.read(int(self.headers["Content-Length"]))), + ) ) - if self.headers.get("User-Agent", "").startswith("python-httpx"): - self.send_response(418) - self.end_headers() - return - response = json.dumps( + response: Final = json.dumps( { "pages": [{"index": 0, "markdown": "native OCR response", "images": [], "dimensions": None}], "model": "mistral-ocr-latest", @@ -38,11 +41,11 @@ def ocr_server(): self.end_headers() self.wfile.write(response) - def log_message(self, format, *args): + def log_message(self, format: str, *args: object) -> None: pass - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - thread = threading.Thread(target=lambda: server.serve_forever(poll_interval=0.01), daemon=True) + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread: Final = threading.Thread(target=lambda: server.serve_forever(poll_interval=0.01), daemon=True) thread.start() try: yield server, requests @@ -52,21 +55,29 @@ def ocr_server(): thread.join() -def test_ocr_with_rust_extension(ocr_server): +def test_native_ocr_with_compiled_rust_extension( + ocr_server: tuple[ThreadingHTTPServer, list[RecordedOCRRequest]], +) -> None: server, requests = ocr_server - host, port = server.server_address + address: Final = server.server_address + host: Final = str(address[0]) + port: Final = int(address[1]) - response = litellm.ocr( - model="mistral/mistral-ocr-latest", + response: Final = rust_ocr_bridge.ocr( + model="mistral-ocr-latest", document={"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, api_key="test-key", api_base=f"http://{host}:{port}", + custom_llm_provider="mistral", + extra_headers=None, + optional_params={}, + timeout=None, ) - assert response.pages[0].markdown == "native OCR response" + assert response is not None + assert response["pages"][0]["markdown"] == "native OCR response" assert len(requests) == 1 - assert not requests[0]["headers"].get("user-agent", "").startswith("python-httpx") - assert requests[0]["body"] == { - "model": "mistral-ocr-latest", - "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, + assert requests[0].body == { + "model": "mistral-ocr-latest", + "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, } From 760043b5337a83f0356cd717e5b555229980deec Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 16:57:53 -0700 Subject: [PATCH 100/126] fix(reset_budget_job): reset end users by budget link, not by user id The cascade zeroed end-user spend with a single update_many whose where clause enumerated every dependent user id. Prisma compiles that IN-list into one prepared statement carrying one bind variable per customer, and PostgreSQL caps a statement at 32,767 of them. Once a shared budget had more dependents than that the statement could not be parsed at all, so the atomic cascade rolled back, budget_reset_at never advanced, and the tier stayed due on every later tick forever. Customers sitting at their cap were blocked indefinitely with only a recurring log line to show for it. End users now match on budget_id like every other gated table, plus a NULL-budget_id branch for the implicitly created rows that carry no link and ride the default tier. The statement's bind count now tracks the number of expiring tiers rather than the customer population, so a reset costs the same whether a budget has ten dependents or a million. Fixes #40564 Claude-Session: https://claude.ai/code/session_01Hn5E8Jz1LjGLFyiYxBRcBW --- .../proxy/common_utils/reset_budget_job.py | 46 ++++++------ .../test_proxy_budget_reset.py | 6 +- .../common_utils/test_reset_budget_job.py | 75 ++++++++++++++++--- 3 files changed, 88 insertions(+), 39 deletions(-) diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index f2648c8466e..35e74418628 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -224,33 +224,31 @@ def _queue_budget_linked_resets( def _queue_enduser_resets(writes: LinkedSpendResetWrites, cascade: "_BudgetCascade") -> None: - """End users are matched by id rather than budget link: rows with no - budget_id ride the default budget tier (litellm.max_end_user_budget_id). - Zero-before-decrement ordering matters here too (see - _queue_budget_linked_resets).""" - if not cascade.rollover_caps: - if cascade.endusers: - writes.queue_spend_zero( - where={"user_id": {"in": [row.user_id for row in cascade.endusers]}} - ) # mutable-ok: prisma where filter must be a dict + """End users reset on the budget link like every other gated table, plus a + NULL-budget_id branch: rows created implicitly persist no link and ride the + default tier (litellm.max_end_user_budget_id). + + Matching on the link rather than enumerating user ids keeps a statement's + bind count proportional to the expiring tiers instead of the customer + population, which past ~32,700 dependents exceeds PostgreSQL's per-statement + bind ceiling and wedges the cascade permanently (#40564). + """ + _queue_budget_linked_resets(writes, cascade, extra=_SPENT_ROWS_WHERE) + default_budget_id: Final = litellm.max_end_user_budget_id + if default_budget_id is None or default_budget_id not in cascade.budget_ids: return - tiered: Final = tuple((row.budget_id or litellm.max_end_user_budget_id, row.user_id) for row in cascade.endusers) - for budget_id, cap in cascade.rollover_caps.items(): - if not ( - user_ids := [uid for bid, uid in tiered if bid == budget_id] - ): # mutable-ok: prisma "in" filter takes a list - continue + cap: Final = cascade.rollover_caps.get(default_budget_id) + if cap is None: writes.queue_spend_zero( - where={"user_id": {"in": user_ids}, "spend": {"lte": cap}} + where={"budget_id": None, **_SPENT_ROWS_WHERE} ) # mutable-ok: prisma where filter must be a dict - writes.queue_spend_decrement( - where={"user_id": {"in": user_ids}, "spend": {"gt": cap}}, amount=cap - ) # mutable-ok: prisma where filter must be a dict - plain: Final = [ - uid for bid, uid in tiered if bid is None or bid not in cascade.rollover_caps - ] # mutable-ok: prisma "in" filter takes a list - if plain: - writes.queue_spend_zero(where={"user_id": {"in": plain}}) # mutable-ok: prisma where filter must be a dict + return + writes.queue_spend_zero( + where={"budget_id": None, "spend": {"gt": 0, "lte": cap}} + ) # mutable-ok: prisma where filter must be a dict + writes.queue_spend_decrement( + where={"budget_id": None, "spend": {"gt": cap}}, amount=cap + ) # mutable-ok: prisma where filter must be a dict @dataclass(frozen=True, slots=True) diff --git a/tests/litellm_utils_tests/test_proxy_budget_reset.py b/tests/litellm_utils_tests/test_proxy_budget_reset.py index 9f6e1f4c3f7..4103536950d 100644 --- a/tests/litellm_utils_tests/test_proxy_budget_reset.py +++ b/tests/litellm_utils_tests/test_proxy_budget_reset.py @@ -401,7 +401,7 @@ async def test_reset_budget_endusers_are_zeroed_with_the_budget_window_advance() enduser_writes = [c for c in batch_calls if c["table"] == "enduser"] assert len(enduser_writes) == 1 - assert enduser_writes[0]["where"]["user_id"]["in"] == [f"user{i}" for i in range(1, 7)] + assert enduser_writes[0]["where"] == {"budget_id": {"in": ["budget1"]}, "spend": {"gt": 0}} assert enduser_writes[0]["data"] == {"spend": 0} budget_writes = [c for c in batch_calls if c["table"] == "budget"] @@ -602,7 +602,7 @@ async def test_reset_budget_continues_other_categories_on_failure(): assert len([c for c in batch_calls if c["table"] == "team_membership"]) == 1 enduser_writes = [c for c in batch_calls if c["table"] == "enduser"] assert len(enduser_writes) == 1 - assert enduser_writes[0]["where"] == {"user_id": {"in": ["user1"]}} + assert enduser_writes[0]["where"] == {"budget_id": {"in": ["budget1"]}, "spend": {"gt": 0}} assert enduser_writes[0]["data"] == {"spend": 0} # Check the new batch write path: 2 keys + 1 user (user1 failed) + 2 teams. @@ -1031,7 +1031,7 @@ async def test_service_logger_endusers_success(): enduser_writes = [c for c in batch_calls if c["table"] == "enduser"] assert len(enduser_writes) == 1 - assert enduser_writes[0]["where"] == {"user_id": {"in": ["user1", "user2"]}} + assert enduser_writes[0]["where"] == {"budget_id": {"in": ["budget1"]}, "spend": {"gt": 0}} proxy_logging_obj.service_logging_obj.async_service_success_hook.assert_called_once() ( diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py index 56c0efb41d2..560953f0b51 100644 --- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py +++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py @@ -403,7 +403,7 @@ def test_reset_budget_for_enduser(reset_budget_job, mock_prisma_client): { "table": "enduser", "op": "update_many", - "where": {"user_id": {"in": ["test-enduser-1"]}}, + "where": {"budget_id": {"in": ["test-budget-1"]}, "spend": {"gt": 0}}, "data": {"spend": 0}, } ] @@ -504,7 +504,7 @@ def test_reset_budget_all(reset_budget_job, mock_prisma_client): { "table": "enduser", "op": "update_many", - "where": {"user_id": {"in": ["test-enduser-1"]}}, + "where": {"budget_id": {"in": ["test-budget-1"]}, "spend": {"gt": 0}}, "data": {"spend": 0}, } ] @@ -524,6 +524,7 @@ _LINKED_TABLE_CASES = [ ("org", {"budget_id": {"in": ["7d-budget-tier"]}, "spend": {"gt": 0}}), ("tag", {"budget_id": {"in": ["7d-budget-tier"]}, "spend": {"gt": 0}}), ("model_access_group", {"budget_id": {"in": ["7d-budget-tier"]}, "spend": {"gt": 0}}), + ("enduser", {"budget_id": {"in": ["7d-budget-tier"]}, "spend": {"gt": 0}}), ] @@ -553,6 +554,48 @@ def test_budget_table_reset_zeroes_spend_on_every_linked_table( assert writes[0]["data"] == {"spend": 0} +_POSTGRES_MAX_BIND_VARIABLES: Final = 32767 + + +def _bind_count(where: Dict[str, Any]) -> int: + """Bind variables one prisma where-clause compiles to: each scalar is one + placeholder and an ``in`` list contributes one per element.""" + return sum(len(value["in"]) if isinstance(value, dict) and "in" in value else 1 for value in where.values()) + + +@pytest.mark.parametrize("population", [3, 40_000], ids=["small", "over-pg-bind-ceiling"]) +def test_enduser_reset_bind_count_does_not_scale_with_population(reset_budget_job, mock_prisma_client, population): + """Regression for #40564. + + Enumerating every dependent user id put one bind variable per customer into + a single prepared statement. Past PostgreSQL's ceiling the statement could + not be parsed at all, so the whole atomic cascade rolled back, + budget_reset_at never advanced, and the tier stayed due on every later tick + forever. Matching on the budget link keeps the statement the same size no + matter how many customers share a tier. + """ + budget = _budget_row(budget_id="shared-tier", budget_duration="1d") + mock_prisma_client.data["budget"] = [budget] + mock_prisma_client.data["enduser"] = [ + types.SimpleNamespace( + spend=1.0, + litellm_budget_table=budget, + user_id=f"cust-{index:08d}", + budget_id="shared-tier", + ) + for index in range(population) + ] + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + writes = _batch_writes(mock_prisma_client, "enduser") + assert [_bind_count(write["where"]) for write in writes] == [2], ( + f"the cascade must not enumerate {population} user ids: past " + f"{_POSTGRES_MAX_BIND_VARIABLES} binds PostgreSQL refuses the statement, got {writes[:1]}" + ) + assert _batch_writes(mock_prisma_client, "budget")[0]["data"]["budget_reset_at"] is not None + + def test_budget_table_reset_writes_nothing_when_no_budget_is_due(reset_budget_job, mock_prisma_client): """Nothing due means no transaction is opened at all.""" asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) @@ -720,14 +763,22 @@ def test_reset_budget_resets_endusers_with_null_budget_id(reset_budget_job, mock asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) - # Both end users are zeroed by the same committed statement. - enduser_writes = _batch_writes(mock_prisma_client, "enduser") - assert len(enduser_writes) == 1, f"Expected a single enduser write, got {enduser_writes}" - assert set(enduser_writes[0]["where"]["user_id"]["in"]) == { - "enduser-explicit", - "enduser-implicit", - } - assert enduser_writes[0]["data"] == {"spend": 0} + # Both end users are zeroed: the linked rows on the tier's budget_id, the + # implicit ones on the NULL branch that stands in for the default tier. + assert _batch_writes(mock_prisma_client, "enduser") == [ + { + "table": "enduser", + "op": "update_many", + "where": {"budget_id": {"in": [default_budget_id]}, "spend": {"gt": 0}}, + "data": {"spend": 0}, + }, + { + "table": "enduser", + "op": "update_many", + "where": {"budget_id": None, "spend": {"gt": 0}}, + "data": {"spend": 0}, + }, + ] # Verify find_many was called to fetch NULL-budget-id end users find_many_calls = mock_prisma_client.db.litellm_endusertable.find_many_calls @@ -3043,13 +3094,13 @@ def test_budget_cascade_carries_enduser_overage_when_rollover_enabled( assert { "table": "enduser", "op": "update_many", - "where": {"user_id": {"in": ["enduser-roll"]}, "spend": {"gt": 10.0}}, + "where": {"budget_id": "budget-roll", "spend": {"gt": 10.0}}, "data": {"spend": {"decrement": 10.0}}, } in enduser_writes assert { "table": "enduser", "op": "update_many", - "where": {"user_id": {"in": ["enduser-roll"]}, "spend": {"lte": 10.0}}, + "where": {"budget_id": "budget-roll", "spend": {"gt": 0, "lte": 10.0}}, "data": {"spend": 0}, } in enduser_writes From ae018825357ddafcf0da15d903d10df0f2581c59 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:14:13 -0700 Subject: [PATCH 101/126] feat(proxy): offload spend tracking to a pod-local collector sidecar (#40545) * feat(proxy): offload spend tracking to a pod-local spend worker sidecar py-spy on the gateway showed the post-response _PROXY_track_cost_callback, spend-log and DBSpendUpdateWriter work running on the inference workers' event loop, so a DB or Redis stall backed up the request path. When LITELLM_SPEND_WORKER_ENABLED=true, _ProxyDBLogger serializes one compact typed SpendEvent per success and hands it to a SpendEventProducer that ships it over a unix socket (default) or loopback-only TCP to a sidecar started as `python -m gateway.spend_worker`. The sidecar runs the unchanged _ProxyDBLogger pipeline against the pod's PgBouncer (pooled_database_url). When the sidecar is unreachable, the buffer is full, or the gateway shuts down with events still queued or in flight, the producer applies LITELLM_SPEND_WORKER_ON_UNAVAILABLE (fallback in-process, or drop). The sidecar half-closes producers on SIGTERM and drains, the producer treats EOF as unavailable, and the gateway flushes buffered spend counters on shutdown. The sidecar honors LITELLM_LOG so its writes are visible in its own process log. Helm: both charts gain an opt-in spend-worker sidecar container sharing an emptyDir socket dir, and the componentized chart's HPA uses a ContainerResource CPU metric scoped to the gateway container so sidecar CPU does not drive inference scaling. * feat(terraform): opt-in spend-worker sidecar for the AWS and GCP gateway stacks Adds spend_worker_* inputs to both modules. On ECS Fargate the sidecar is a second, non-essential container in the gateway task; on Cloud Run it is a second container in the gateway service. Both listen on loopback TCP, share the gateway's DB/Redis/secret env, and set LITELLM_JOB_ROLE=spend_worker. Disabled by default. Plan-only tests cover both, and the terraform CI workflow now runs the gcp module too Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): retrieve a completed batch in the in-process spend path test The base now defers cost tracking for batches that are still in flight, so an in_progress batch never reaches update_database Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): rename the spend worker sidecar to collector Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): run the collector from the installed litellm package and finish in-flight fallbacks on shutdown The sidecar command becomes python -m litellm.proxy.collector so the classic image, whose runtime stage copies only the installed package, can run it. The module now assembles DATABASE_URL and the pod-local pgbouncer URL itself, replacing gateway/collector.py The componentized collector sidecar inherits gateway.volumeMounts so custom CA mounts reach it. SpendEventProducer shields an in-progress fallback from the writer task cancellation so close() no longer loses an event already handed to the in-process pipeline Helpers used across modules (address_argument, should_store_prompts_and_responses_in_spend_logs, flush_spend_counters_on_shutdown) become public so the change adds no reportPrivateUsage errors Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci(terraform): drop the gcp job duplicated by the aws/gcp matrix Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(collector): keep metrics env off the classic sidecar and reject shared loopback ports The classic chart no longer hands PROMETHEUS_METRICS_PORT and the billing metrics env to the collector container, and gives it the same /.npm scratch mount as the proxy on a read-only root. AWS and GCP now refuse a plan where the spend collector and the metrics sidecar bind the same loopback port. A regression test drives a sidecar crash mid-stream on asyncio and uvloop and checks no event is billed by both the sidecar and the in-process fallback; the producer docstring spells out why a failed drain() cannot double count Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(proxy): format pooled_database_url after the pgbouncer rebase Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep the cache-hit preset key and survive dead producers on collector drain Cache hits updated the logging object after the early return, so the offloaded spend event carried preset_cache_key=None and the collector re-hashed reconstructed kwargs. Also guard write_eof() against producer transports uvloop already closed so one dead connection cannot abort the drain Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(terraform): keep the gcp collector port off the metrics sidecar health port Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): collector connects to Postgres directly under IAM or Entra token auth Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): mark the collector's DATABASE_URL as pooled when it uses the pod's pgbouncer Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-terraform-modules.yml | 40 +- helm/litellm-helm/templates/_helpers.tpl | 160 +++++++ helm/litellm-helm/templates/deployment.yaml | 180 +++----- helm/litellm-helm/templates/hpa.yaml | 10 + helm/litellm-helm/tests/collector_tests.yaml | 272 ++++++++++++ helm/litellm-helm/values.yaml | 42 ++ helm/litellm/templates/_helpers.tpl | 31 ++ .../litellm/templates/gateway/deployment.yaml | 56 ++- helm/litellm/templates/gateway/hpa.yaml | 10 + helm/litellm/tests/collector_tests.yaml | 204 +++++++++ helm/litellm/values.yaml | 36 ++ litellm/caching/caching_handler.py | 4 +- litellm/proxy/collector.py | 220 +++++++++ litellm/proxy/db/pgbouncer.py | 16 + .../proxy/hooks/proxy_track_cost_callback.py | 54 ++- litellm/proxy/proxy_server.py | 39 +- litellm/proxy/spend_tracking/spend_event.py | 418 ++++++++++++++++++ .../spend_tracking/spend_event_producer.py | 338 ++++++++++++++ .../spend_tracking/spend_tracking_utils.py | 24 +- litellm/utils.py | 3 + terraform/litellm/README.md | 1 + terraform/litellm/aws/README.md | 33 ++ terraform/litellm/aws/ecs.tf | 82 +++- .../litellm/aws/tests/collector.tftest.hcl | 144 ++++++ terraform/litellm/aws/variables.tf | 70 +++ terraform/litellm/gcp/README.md | 37 ++ terraform/litellm/gcp/cloudrun.tf | 85 +++- .../litellm/gcp/tests/collector.tftest.hcl | 165 +++++++ terraform/litellm/gcp/variables.tf | 70 +++ tests/test_gateway/test_launch.py | 6 +- .../caching/test_caching_handler.py | 35 ++ .../hooks/test_proxy_track_cost_callback.py | 255 ++++++++++- .../proxy/proxy_server/test_lifecycle.py | 32 ++ .../proxy/spend_tracking/test_spend_event.py | 213 +++++++++ .../test_spend_event_producer.py | 359 +++++++++++++++ .../test_spend_tracking_utils.py | 122 +++-- tests/test_litellm/proxy/test_collector.py | 229 ++++++++++ 37 files changed, 3865 insertions(+), 230 deletions(-) create mode 100644 helm/litellm-helm/tests/collector_tests.yaml create mode 100644 helm/litellm/tests/collector_tests.yaml create mode 100644 litellm/proxy/collector.py create mode 100644 litellm/proxy/spend_tracking/spend_event.py create mode 100644 litellm/proxy/spend_tracking/spend_event_producer.py create mode 100644 terraform/litellm/aws/tests/collector.tftest.hcl create mode 100644 terraform/litellm/gcp/tests/collector.tftest.hcl create mode 100644 tests/test_litellm/proxy/spend_tracking/test_spend_event.py create mode 100644 tests/test_litellm/proxy/spend_tracking/test_spend_event_producer.py create mode 100644 tests/test_litellm/proxy/test_collector.py diff --git a/.github/workflows/test-terraform-modules.yml b/.github/workflows/test-terraform-modules.yml index 52006d9b578..e6896604b7f 100644 --- a/.github/workflows/test-terraform-modules.yml +++ b/.github/workflows/test-terraform-modules.yml @@ -25,13 +25,17 @@ concurrency: cancel-in-progress: true jobs: - aws-module: - name: fmt, validate, test (aws) + module: + name: fmt, validate, test (${{ matrix.module }}) runs-on: ubuntu-latest timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + module: [aws, gcp] defaults: run: - working-directory: terraform/litellm/aws + working-directory: terraform/litellm/${{ matrix.module }} steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: @@ -51,35 +55,7 @@ jobs: - name: validate run: terraform validate - # Plan-only, mock_provider-backed: no AWS credentials, no API calls. + # Plan-only, mock_provider-backed: no cloud credentials, no API calls. - name: test run: terraform test - gcp-module: - name: fmt, validate, test (gcp) - runs-on: ubuntu-latest - timeout-minutes: 15 - defaults: - run: - working-directory: terraform/litellm/gcp - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 - with: - terraform_version: 1.13.3 - terraform_wrapper: false - - - name: fmt - run: terraform fmt -recursive -check -diff - - - name: init - run: terraform init -backend=false -input=false - - - name: validate - run: terraform validate - - - name: test - run: terraform test diff --git a/helm/litellm-helm/templates/_helpers.tpl b/helm/litellm-helm/templates/_helpers.tpl index 8f2acb20fce..9630633912e 100644 --- a/helm/litellm-helm/templates/_helpers.tpl +++ b/helm/litellm-helm/templates/_helpers.tpl @@ -161,3 +161,163 @@ taken before the change, which by that point no longer exists. {{- fail (printf "postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got %q). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore." $tag) -}} {{- end -}} {{- end -}} + +{{/* +Environment shared by the proxy container and the opt-in collector sidecar: +database, pgbouncer, master key, redis, user envVars. Both containers must see +the same DATABASE_URL and REDIS_* so the sidecar reaches the pod's pgbouncer +and the same spend transaction buffer. +*/}} +{{- define "litellm.proxyEnv" -}} +- name: HOST + value: "{{ .Values.listen | default "0.0.0.0" }}" +- name: PORT + value: {{ .Values.service.port | quote}} +{{- if .Values.db.deployStandalone }} +- name: DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: {{ include "litellm.fullname" . }}-dbcredentials + key: username +- name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "litellm.fullname" . }}-dbcredentials + key: password +- name: DATABASE_HOST + value: {{ .Release.Name }}-postgresql +- name: DATABASE_NAME + value: litellm +{{- else if .Values.db.useExisting }} +- name: DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.usernameKey }} +- name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.passwordKey }} +- name: DATABASE_HOST + {{- if .Values.db.secret.endpointKey }} + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.endpointKey }} + {{- else }} + value: {{ .Values.db.endpoint }} + {{- end }} +- name: DATABASE_NAME + value: {{ .Values.db.database }} +- name: DATABASE_URL + value: {{ .Values.db.url | quote }} +{{- end }} +{{- if and .Values.db.useExisting .Values.db.readReplicaUrl .Values.db.secret.readReplicaEndpointKey (not .Values.db.secret.readReplicaUrlKey) }} +- name: DATABASE_READER_HOST + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.readReplicaEndpointKey }} +{{- end }} +{{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }} +- name: DATABASE_URL_READ_REPLICA + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.readReplicaUrlKey }} +{{- else if .Values.db.readReplicaUrl }} +- name: DATABASE_URL_READ_REPLICA + value: {{ .Values.db.readReplicaUrl | quote }} +{{- end }} +{{- if .Values.db.connectionPool.enabled }} +- name: LITELLM_PGBOUNCER_ENABLED + value: "true" +- name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: {{ .Values.db.connectionPool.maxDbConnections | quote }} +- name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + value: {{ .Values.db.connectionPool.maxClientConn | quote }} +{{- end }} +- name: PROXY_MASTER_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.masterkeySecretName | default (printf "%s-masterkey" (include "litellm.fullname" .)) }} + key: {{ .Values.masterkeySecretKey | default "masterkey" }} +{{- if .Values.redis.enabled }} +- name: REDIS_HOST + value: {{ include "litellm.redis.serviceName" . }} +- name: REDIS_PORT + value: {{ include "litellm.redis.port" . | quote }} +- name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "redis.secretName" .Subcharts.redis }} + key: {{include "redis.secretPasswordKey" .Subcharts.redis }} +{{- end }} +{{- /* + Inject LITELLM_LOG only when envVars does not already define it. +*/}} +{{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }} +- name: LITELLM_LOG + value: {{ .Values.logLevel | quote }} +{{- end }} +{{- if .Values.envVars }} +{{- range $key, $val := .Values.envVars }} +- name: {{ $key }} + value: {{ $val | quote }} +{{- end }} +{{- end }} +{{- with .Values.extraEnvVars }} +{{ toYaml . }} +{{- end }} +{{- if .Values.migrationJob.enabled }} +# Schema updates are owned by the dedicated migrations Job; skip +# the proxy's startup `prisma db push` so N replicas don't race +# one DB on every rollout. Placed last (after envVars and +# extraEnvVars) so this override can't be silently shadowed by a +# user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env +# semantics — same pattern the migrations Job uses. +- name: DISABLE_SCHEMA_UPDATE + value: "true" +{{- end }} +{{- end -}} + +{{/* +Proxy-only metering and metrics env. The collector sidecar serves no HTTP +traffic, so it gets neither. +*/}} +{{- define "litellm.proxyMetricsEnv" -}} +{{- if .Values.billingMetrics.enabled }} +{{ include "litellm.billingMetricsEnv" . }} +{{- end }} +{{- if .Values.metricsServer.enabled }} +{{- if eq (int .Values.metricsServer.port) (int .Values.service.port) }} +{{- fail "metricsServer.port must differ from service.port" }} +{{- end }} +- name: PROMETHEUS_METRICS_PORT + value: {{ .Values.metricsServer.port | quote }} +{{- end }} +{{- end -}} + +{{/* +Directory of the collector's unix socket, shared between the two containers +through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP. +*/}} +{{- define "litellm.collector.socketDir" -}} +{{- if and .Values.collector.enabled (hasPrefix "unix://" .Values.collector.address) -}} +{{- dir (trimPrefix "unix://" .Values.collector.address) -}} +{{- end -}} +{{- end -}} + +{{- define "litellm.collectorEnv" -}} +- name: LITELLM_COLLECTOR_ENABLED + value: "true" +- name: LITELLM_COLLECTOR_ADDRESS + value: {{ .Values.collector.address | quote }} +- name: LITELLM_COLLECTOR_BUFFER_SIZE + value: {{ .Values.collector.bufferSize | quote }} +- name: LITELLM_COLLECTOR_ON_UNAVAILABLE + value: {{ .Values.collector.onUnavailable | quote }} +- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS + value: {{ .Values.collector.drainTimeoutSeconds | quote }} +{{- end -}} diff --git a/helm/litellm-helm/templates/deployment.yaml b/helm/litellm-helm/templates/deployment.yaml index 834071eb9b2..cf7b3f8a38d 100644 --- a/helm/litellm-helm/templates/deployment.yaml +++ b/helm/litellm-helm/templates/deployment.yaml @@ -56,126 +56,10 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} env: - - name: HOST - value: "{{ .Values.listen | default "0.0.0.0" }}" - - name: PORT - value: {{ .Values.service.port | quote}} - {{- if .Values.db.deployStandalone }} - - name: DATABASE_USERNAME - valueFrom: - secretKeyRef: - name: {{ include "litellm.fullname" . }}-dbcredentials - key: username - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "litellm.fullname" . }}-dbcredentials - key: password - - name: DATABASE_HOST - value: {{ .Release.Name }}-postgresql - - name: DATABASE_NAME - value: litellm - {{- else if .Values.db.useExisting }} - - name: DATABASE_USERNAME - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.usernameKey }} - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.passwordKey }} - - name: DATABASE_HOST - {{- if .Values.db.secret.endpointKey }} - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.endpointKey }} - {{- else }} - value: {{ .Values.db.endpoint }} - {{- end }} - - name: DATABASE_NAME - value: {{ .Values.db.database }} - - name: DATABASE_URL - value: {{ .Values.db.url | quote }} - {{- end }} - {{- if and .Values.db.useExisting .Values.db.readReplicaUrl .Values.db.secret.readReplicaEndpointKey (not .Values.db.secret.readReplicaUrlKey) }} - - name: DATABASE_READER_HOST - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.readReplicaEndpointKey }} - {{- end }} - {{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }} - - name: DATABASE_URL_READ_REPLICA - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.readReplicaUrlKey }} - {{- else if .Values.db.readReplicaUrl }} - - name: DATABASE_URL_READ_REPLICA - value: {{ .Values.db.readReplicaUrl | quote }} - {{- end }} - {{- if .Values.db.connectionPool.enabled }} - - name: LITELLM_PGBOUNCER_ENABLED - value: "true" - - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - value: {{ .Values.db.connectionPool.maxDbConnections | quote }} - - name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN - value: {{ .Values.db.connectionPool.maxClientConn | quote }} - {{- end }} - - name: PROXY_MASTER_KEY - valueFrom: - secretKeyRef: - name: {{ .Values.masterkeySecretName | default (printf "%s-masterkey" (include "litellm.fullname" .)) }} - key: {{ .Values.masterkeySecretKey | default "masterkey" }} - {{- if .Values.redis.enabled }} - - name: REDIS_HOST - value: {{ include "litellm.redis.serviceName" . }} - - name: REDIS_PORT - value: {{ include "litellm.redis.port" . | quote }} - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "redis.secretName" .Subcharts.redis }} - key: {{include "redis.secretPasswordKey" .Subcharts.redis }} - {{- end }} - {{- /* - Inject LITELLM_LOG only when envVars does not already define it. - */}} - {{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }} - - name: LITELLM_LOG - value: {{ .Values.logLevel | quote }} - {{- end }} - {{- if .Values.envVars }} - {{- range $key, $val := .Values.envVars }} - - name: {{ $key }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with .Values.extraEnvVars }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsEnv" . | nindent 12 }} - {{- end }} - {{- if .Values.metricsServer.enabled }} - {{- if eq (int .Values.metricsServer.port) (int .Values.service.port) }} - {{- fail "metricsServer.port must differ from service.port" }} - {{- end }} - - name: PROMETHEUS_METRICS_PORT - value: {{ .Values.metricsServer.port | quote }} - {{- end }} - {{- if .Values.migrationJob.enabled }} - # Schema updates are owned by the dedicated migrations Job; skip - # the proxy's startup `prisma db push` so N replicas don't race - # one DB on every rollout. Placed last (after envVars and - # extraEnvVars) so this override can't be silently shadowed by a - # user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env - # semantics — same pattern the migrations Job uses. - - name: DISABLE_SCHEMA_UPDATE - value: "true" + {{- include "litellm.proxyEnv" . | nindent 12 }} + {{- include "litellm.proxyMetricsEnv" . | nindent 12 }} + {{- if .Values.collector.enabled }} + {{- include "litellm.collectorEnv" . | nindent 12 }} {{- end }} envFrom: {{- range .Values.environmentSecrets }} @@ -253,6 +137,10 @@ spec: {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }} {{- end }} + {{- if include "litellm.collector.socketDir" . }} + - name: collector-socket + mountPath: {{ include "litellm.collector.socketDir" . }} + {{- end }} {{- with .Values.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} @@ -260,6 +148,53 @@ spec: lifecycle: {{- toYaml . | nindent 12 }} {{- end }} + {{- if .Values.collector.enabled }} + - name: {{ include "litellm.name" . }}-collector + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: {{ toYaml .Values.collector.command | nindent 12 }} + env: + {{- include "litellm.proxyEnv" . | nindent 12 }} + {{- include "litellm.collectorEnv" . | nindent 12 }} + - name: LITELLM_JOB_ROLE + value: collector + {{- if not (hasKey (default dict .Values.envVars) "CONFIG_FILE_PATH") }} + - name: CONFIG_FILE_PATH + value: /etc/litellm/config.yaml + {{- end }} + envFrom: + {{- range .Values.environmentSecrets }} + - secretRef: + name: {{ . }} + {{- end }} + {{- range .Values.environmentConfigMaps }} + - configMapRef: + name: {{ . }} + {{- end }} + resources: + {{- toYaml .Values.collector.resources | nindent 12 }} + volumeMounts: + - name: litellm-config + mountPath: /etc/litellm/config.yaml + subPath: config.yaml + {{- if include "litellm.collector.socketDir" . }} + - name: collector-socket + mountPath: {{ include "litellm.collector.socketDir" . }} + {{- end }} + {{ if .Values.securityContext.readOnlyRootFilesystem }} + - name: tmp + mountPath: /tmp + - name: cache + mountPath: /.cache + - name: npm + mountPath: /.npm + {{- end }} + {{- with .Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} {{- with .Values.extraContainers }} {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} @@ -288,6 +223,11 @@ spec: {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsVolumes" . | nindent 8 }} {{- end }} + {{- if include "litellm.collector.socketDir" . }} + - name: collector-socket + emptyDir: + sizeLimit: 1Mi + {{- end }} {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/helm/litellm-helm/templates/hpa.yaml b/helm/litellm-helm/templates/hpa.yaml index 010c4095ab3..a651f916d21 100644 --- a/helm/litellm-helm/templates/hpa.yaml +++ b/helm/litellm-helm/templates/hpa.yaml @@ -18,6 +18,15 @@ spec: {{- end }} metrics: {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- if and .Values.collector.enabled .Values.collector.scaleOnProxyContainerCpu }} + - type: ContainerResource + containerResource: + name: cpu + container: {{ include "litellm.name" . }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- else }} - type: Resource resource: name: cpu @@ -25,6 +34,7 @@ spec: type: Utilization averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} + {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: diff --git a/helm/litellm-helm/tests/collector_tests.yaml b/helm/litellm-helm/tests/collector_tests.yaml new file mode 100644 index 00000000000..0340b1161b7 --- /dev/null +++ b/helm/litellm-helm/tests/collector_tests.yaml @@ -0,0 +1,272 @@ +suite: test collector sidecar +templates: + - deployment.yaml + - hpa.yaml + - configmap-litellm.yaml +tests: + - it: should run the proxy alone with no collector env by default + template: deployment.yaml + asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 1 + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ENABLED + value: "true" + - notContains: + path: spec.template.spec.volumes + content: + name: collector-socket + any: true + + - it: should add the sidecar on the same image and point both containers at the unix socket + template: deployment.yaml + set: + image.tag: test + db.connectionPool.enabled: true + collector.enabled: true + collector.resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 2 + - equal: + path: spec.template.spec.containers[1].name + value: litellm-collector + - equal: + path: spec.template.spec.containers[1].image + value: ghcr.io/berriai/litellm:test + - equal: + path: spec.template.spec.containers[1].command + value: [python, -m, litellm.proxy.collector] + - equal: + path: spec.template.spec.containers[1].resources.requests.cpu + value: 500m + - equal: + path: spec.template.spec.containers[1].resources.limits.memory + value: 2Gi + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ENABLED + value: "true" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ADDRESS + value: unix:///var/run/litellm/collector.sock + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_BUFFER_SIZE + value: "1000" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ON_UNAVAILABLE + value: fallback + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_JOB_ROLE + value: collector + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_JOB_ROLE + value: collector + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_COLLECTOR_ADDRESS + value: unix:///var/run/litellm/collector.sock + - contains: + path: spec.template.spec.containers[1].env + content: + name: CONFIG_FILE_PATH + value: /etc/litellm/config.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: DATABASE_HOST + value: RELEASE-NAME-postgresql + - contains: + path: spec.template.spec.containers[1].env + content: + name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: RELEASE-NAME-litellm-dbcredentials + key: password + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_PGBOUNCER_ENABLED + value: "true" + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: "20" + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: collector-socket + mountPath: /var/run/litellm + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: collector-socket + mountPath: /var/run/litellm + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: litellm-config + mountPath: /etc/litellm/config.yaml + subPath: config.yaml + - contains: + path: spec.template.spec.volumes + content: + name: collector-socket + emptyDir: + sizeLimit: 1Mi + + - it: should skip the socket volume and pass the policy through on tcp transport + template: deployment.yaml + set: + collector.enabled: true + collector.address: tcp://127.0.0.1:4100 + collector.onUnavailable: drop + collector.bufferSize: 50 + envVars: + CONFIG_FILE_PATH: /custom/config.yaml + asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 2 + - notContains: + path: spec.template.spec.containers[1].env + content: + name: CONFIG_FILE_PATH + value: /etc/litellm/config.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: CONFIG_FILE_PATH + value: /custom/config.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ADDRESS + value: tcp://127.0.0.1:4100 + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ON_UNAVAILABLE + value: drop + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_BUFFER_SIZE + value: "50" + - notContains: + path: spec.template.spec.volumes + content: + name: collector-socket + any: true + + - it: should keep metrics and billing env on the proxy container only + template: deployment.yaml + set: + collector.enabled: true + metricsServer.enabled: true + metricsServer.port: 9090 + billingMetrics.enabled: true + billingMetrics.endpoint: https://metering.example.com + billingMetrics.secretName: billing-mtls + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROMETHEUS_METRICS_PORT + value: "9090" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_BILLING_METRICS_ENDPOINT + value: https://metering.example.com + - notContains: + path: spec.template.spec.containers[1].env + content: + name: PROMETHEUS_METRICS_PORT + any: true + - notContains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_BILLING_METRICS_ENDPOINT + any: true + - notContains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: billing-metrics-mtls + any: true + + - it: should give the sidecar the same scratch mounts as the proxy on a read-only root + template: deployment.yaml + set: + collector.enabled: true + securityContext.readOnlyRootFilesystem: true + asserts: + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: npm + mountPath: /.npm + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: cache + mountPath: /.cache + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: tmp + mountPath: /tmp + + - it: should keep the pod-wide cpu metric unless asked to scale on the proxy container + template: hpa.yaml + set: + autoscaling.enabled: true + collector.enabled: true + asserts: + - equal: { path: "spec.metrics[0].type", value: Resource } + - equal: { path: "spec.metrics[0].resource.name", value: cpu } + + - it: should scale on the proxy container's cpu only when opted in + template: hpa.yaml + set: + autoscaling.enabled: true + collector.enabled: true + collector.scaleOnProxyContainerCpu: true + asserts: + - equal: { path: "spec.metrics[0].type", value: ContainerResource } + - equal: { path: "spec.metrics[0].containerResource.name", value: cpu } + - equal: { path: "spec.metrics[0].containerResource.container", value: litellm } + - equal: { path: "spec.metrics[0].containerResource.target.averageUtilization", value: 60 } + - isNull: { path: "spec.metrics[0].resource" } + + - it: should not switch to the container metric while the sidecar is off + template: hpa.yaml + set: + autoscaling.enabled: true + collector.scaleOnProxyContainerCpu: true + asserts: + - equal: { path: "spec.metrics[0].type", value: Resource } diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index db596e2d68e..fcee331a5aa 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -190,6 +190,48 @@ metricsServer: enabled: false port: 4001 +# Opt-in sidecar that runs the post-response spend pipeline (cost calculation, +# spend logs, spend counters, budget reservation reconciliation) so the proxy's +# uvicorn workers only serialise a compact typed event and go back to serving +# inference. Same image and tag as the proxy, second container in the same pod, +# fed over loopback (a unix socket on a shared emptyDir, or 127.0.0.1 TCP). It +# reuses the pod's in-container pgbouncer (db.connectionPool) and the same Redis +# spend transaction buffer, so the per-pod DB connection budget is unchanged. +# Delivery is at-most-once inside the pod: events already handed to the sidecar +# are lost if it crashes before writing them; events the workers could not hand +# over follow onUnavailable. Both containers drain on SIGTERM within +# terminationGracePeriodSeconds +collector: + enabled: false + # unix:////.sock (the becomes a shared emptyDir) or tcp://127.0.0.1: + address: unix:///var/run/litellm/collector.sock + # Events each uvicorn worker holds in memory while the sidecar is slow or restarting + bufferSize: 1000 + # fallback: run the pipeline in the worker when the sidecar is unreachable or the + # buffer is full (spend stays exact, that request costs proxy CPU again) + # drop: count and discard the event instead (spend under-reports) + onUnavailable: fallback + # How long the workers keep pushing buffered events on shutdown, and how long the + # sidecar keeps serving its open connections after SIGTERM + drainTimeoutSeconds: 10 + command: + - python + - -m + - litellm.proxy.collector + # Sized independently of the proxy container; the pipeline is CPU bound + resources: {} + # requests: + # cpu: 500m + # memory: 1Gi + # limits: + # cpu: "1" + # memory: 2Gi + # When autoscaling.enabled, swap the pod-wide cpu Resource metric for an + # autoscaling/v2 ContainerResource metric on the proxy container only, so the + # sidecar's CPU never scales inference replicas. Needs Kubernetes 1.30+ (or the + # HPAContainerMetrics feature gate on 1.27 to 1.29) + scaleOnProxyContainerCpu: false + resources: {} # Unset by default so the chart installs on small clusters such as Minikube, and so an diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index dc47bc0b0a1..692a799e783 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -457,3 +457,34 @@ ImplementationSpecific {{- end -}} {{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}} + +{{/* +Directory of the collector's unix socket, shared by the gateway and +collector containers through an emptyDir. Empty when the sidecar is off +or gateway.collector.address is a tcp://127.0.0.1: address. +*/}} +{{- define "litellm.gateway.collectorSocketDir" -}} +{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}} +{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}} +{{- end -}} +{{- end -}} + +{{/* +LITELLM_COLLECTOR_* env shared by the producer (gateway container) and the +consumer (collector container), so both agree on the transport and the +shutdown drain window. +*/}} +{{- define "litellm.gateway.collectorEnv" -}} +{{- with .Values.gateway.collector }} +- name: LITELLM_COLLECTOR_ENABLED + value: "true" +- name: LITELLM_COLLECTOR_ADDRESS + value: {{ .address | quote }} +- name: LITELLM_COLLECTOR_BUFFER_SIZE + value: {{ .bufferSize | quote }} +- name: LITELLM_COLLECTOR_ON_UNAVAILABLE + value: {{ .onUnavailable | quote }} +- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS + value: {{ .drainTimeoutSeconds | quote }} +{{- end }} +{{- end -}} diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index 1e9041f0a33..5c1a089b50a 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -74,8 +74,11 @@ spec: - name: PROMETHEUS_MULTIPROC_DIR value: {{ include "litellm.gateway.prometheusMultiprocDir" . }} {{- end }} + {{- if .Values.gateway.collector.enabled }} + {{- include "litellm.gateway.collectorEnv" . | nindent 12 }} + {{- end }} {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} - {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }} + {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} volumeMounts: {{- if .Values.gateway.config.create }} - name: gateway-config @@ -86,6 +89,10 @@ spec: - name: prometheus-multiproc mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }} {{- end }} + {{- if include "litellm.gateway.collectorSocketDir" . }} + - name: collector-socket + mountPath: {{ include "litellm.gateway.collectorSocketDir" . }} + {{- end }} {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }} {{- end }} @@ -145,10 +152,50 @@ spec: resources: {{- toYaml .Values.gateway.metricsServer.resources | nindent 12 }} {{- end }} + {{- if .Values.gateway.collector.enabled }} + - name: collector + image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.gateway.image.pullPolicy }} + {{- with .Values.gateway.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + command: + - python + - -m + - litellm.proxy.collector + env: + {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }} + {{- if .Values.gateway.config.create }} + - name: CONFIG_FILE_PATH + value: /app/config/config.yaml + {{- end }} + {{- include "litellm.gateway.collectorEnv" . | nindent 12 }} + - name: LITELLM_JOB_ROLE + value: collector + {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} + {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts (include "litellm.gateway.collectorSocketDir" .) }} + volumeMounts: + {{- if .Values.gateway.config.create }} + - name: gateway-config + mountPath: /app/config/config.yaml + subPath: config.yaml + {{- end }} + {{- if include "litellm.gateway.collectorSocketDir" . }} + - name: collector-socket + mountPath: {{ include "litellm.gateway.collectorSocketDir" . }} + {{- end }} + {{- with .Values.gateway.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + resources: + {{- toYaml .Values.gateway.collector.resources | nindent 12 }} + {{- end }} {{- with .Values.gateway.extraContainers }} {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} - {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }} + {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} volumes: {{- if .Values.gateway.config.create }} - name: gateway-config @@ -159,6 +206,11 @@ spec: - name: prometheus-multiproc emptyDir: {} {{- end }} + {{- if include "litellm.gateway.collectorSocketDir" . }} + - name: collector-socket + emptyDir: + sizeLimit: 1Mi + {{- end }} {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsVolumes" . | nindent 8 }} {{- end }} diff --git a/helm/litellm/templates/gateway/hpa.yaml b/helm/litellm/templates/gateway/hpa.yaml index 4183079cd31..e7094e96106 100644 --- a/helm/litellm/templates/gateway/hpa.yaml +++ b/helm/litellm/templates/gateway/hpa.yaml @@ -15,6 +15,15 @@ spec: maxReplicas: {{ .Values.gateway.hpa.maxReplicas }} metrics: {{- if .Values.gateway.hpa.targetCPUUtilizationPercentage }} + {{- if and .Values.gateway.collector.enabled .Values.gateway.collector.scaleOnGatewayContainerCpu }} + - type: ContainerResource + containerResource: + name: cpu + container: gateway + target: + type: Utilization + averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }} + {{- else }} - type: Resource resource: name: cpu @@ -22,6 +31,7 @@ spec: type: Utilization averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }} {{- end }} + {{- end }} {{- if .Values.gateway.hpa.targetMemoryUtilizationPercentage }} - type: Resource resource: diff --git a/helm/litellm/tests/collector_tests.yaml b/helm/litellm/tests/collector_tests.yaml new file mode 100644 index 00000000000..b1199a16793 --- /dev/null +++ b/helm/litellm/tests/collector_tests.yaml @@ -0,0 +1,204 @@ +suite: test gateway collector sidecar +templates: + - gateway/configmap.yaml + - gateway/deployment.yaml + - gateway/hpa.yaml +values: + - ./values/required.yaml +tests: + - it: adds no sidecar, env, volume or container metric when the collector is off + asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 1 + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ENABLED + value: "true" + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.volumes + content: + name: collector-socket + any: true + template: gateway/deployment.yaml + - equal: + path: spec.metrics[0].type + value: Resource + template: gateway/hpa.yaml + + - it: runs the collector as a sidecar sharing env, config and a unix socket emptyDir, and scales on the gateway container only + set: + gateway.collector.enabled: true + gateway.collector.bufferSize: 250 + gateway.collector.onUnavailable: drop + gateway.image.tag: v1.102.0 + gateway.numWorkers: 4 + gateway.extraEnv: + - name: LITELLM_PGBOUNCER_ENABLED + value: "true" + gateway.envSecrets: + - litellm-license + gateway.volumes: + - name: redis-ca + secret: + secretName: redis-ca + gateway.volumeMounts: + - name: redis-ca + mountPath: /etc/litellm/redis-ca + readOnly: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ADDRESS + value: unix:///var/run/litellm/collector.sock + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_BUFFER_SIZE + value: "250" + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ON_UNAVAILABLE + value: drop + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: collector-socket + mountPath: /var/run/litellm + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].name + value: collector + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].image + value: ghcr.io/berriai/litellm-gateway:v1.102.0 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].command + value: + - python + - -m + - litellm.proxy.collector + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_JOB_ROLE + value: collector + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: CONFIG_FILE_PATH + value: /app/config/config.yaml + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_PGBOUNCER_ENABLED + value: "true" + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: DATABASE_HOST + value: postgres.example.com + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_COLLECTOR_ADDRESS + value: unix:///var/run/litellm/collector.sock + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.containers[1].env + content: + name: NUM_WORKERS + any: true + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].envFrom + value: + - secretRef: + name: litellm-license + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: gateway-config + mountPath: /app/config/config.yaml + subPath: config.yaml + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: collector-socket + mountPath: /var/run/litellm + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: redis-ca + mountPath: /etc/litellm/redis-ca + readOnly: true + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].resources.limits.cpu + value: "1" + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.volumes + content: + name: collector-socket + emptyDir: + sizeLimit: 1Mi + template: gateway/deployment.yaml + - equal: + path: spec.metrics[0] + value: + type: ContainerResource + containerResource: + name: cpu + container: gateway + target: + type: Utilization + averageUtilization: 70 + template: gateway/hpa.yaml + + - it: uses loopback tcp without a socket volume and keeps the pod-wide cpu metric when asked + set: + gateway.collector.enabled: true + gateway.collector.address: tcp://127.0.0.1:4010 + gateway.collector.scaleOnGatewayContainerCpu: false + asserts: + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_COLLECTOR_ADDRESS + value: tcp://127.0.0.1:4010 + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.volumes + content: + name: collector-socket + any: true + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: collector-socket + any: true + template: gateway/deployment.yaml + - equal: + path: spec.metrics[0].type + value: Resource + template: gateway/hpa.yaml diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 677e21a5a51..1f67b984513 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -314,6 +314,42 @@ gateway: labels: {} interval: 15s scrapeTimeout: 10s + # Opt-in `collector` sidecar (same image, `python -m litellm.proxy.collector`) + # that runs the post-response spend pipeline (cost calculation, spend logs, + # spend counters, budget reservation reconciliation) so the uvicorn workers + # only serialise a compact event over loopback and go back to serving + # requests. It shares the pod's env, proxy config, in-container pgbouncer and + # Redis spend buffer, so the per-pod DB connection budget is unchanged. + # Delivery is at-most-once inside the pod: events already handed over are + # lost if the sidecar dies before writing them; events the workers cannot + # hand over follow `onUnavailable`. + collector: + enabled: false + # unix:////.sock (the becomes a shared emptyDir) or + # tcp://127.0.0.1: + address: unix:///var/run/litellm/collector.sock + # Events each uvicorn worker holds in memory while the sidecar is slow or + # restarting. + bufferSize: 1000 + # fallback: run the pipeline in the worker when the sidecar is unreachable + # or the buffer is full (spend stays exact, that request costs gateway CPU + # again). drop: count and discard the event instead (spend under-reports). + onUnavailable: fallback + # How long the workers keep pushing buffered events on shutdown, and how + # long the sidecar keeps serving open connections after SIGTERM. + drainTimeoutSeconds: 10 + # Sized independently of the gateway container; the pipeline is CPU bound. + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + # With hpa.targetCPUUtilizationPercentage set, scale on an autoscaling/v2 + # ContainerResource metric of the `gateway` container only, so the + # sidecar's CPU never drives inference replicas. Needs Kubernetes 1.30+. + scaleOnGatewayContainerCpu: true image: repository: ghcr.io/berriai/litellm-gateway tag: "" # defaults to .Chart.AppVersion diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 0de88eacaa5..139dcf058d2 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -1217,7 +1217,9 @@ class LLMCachingHandler: } if litellm.cache is not None: - litellm_params["preset_cache_key"] = litellm.cache._get_preset_cache_key_from_kwargs(**kwargs) + litellm_params["preset_cache_key"] = ( + self.preset_cache_key or litellm.cache._get_preset_cache_key_from_kwargs(**kwargs) + ) else: litellm_params["preset_cache_key"] = None diff --git a/litellm/proxy/collector.py b/litellm/proxy/collector.py new file mode 100644 index 00000000000..3ff2a83860b --- /dev/null +++ b/litellm/proxy/collector.py @@ -0,0 +1,220 @@ +"""Collector sidecar: consume spend events from the pod's inference workers and run the cost pipeline. + +Runs the proxy startup lifespan (config, Prisma, Redis transaction buffer, scheduled spend flushes) +without serving HTTP, then listens on ``LITELLM_COLLECTOR_ADDRESS`` for newline-delimited spend +events. Each event goes through the unchanged ``_ProxyDBLogger._PROXY_track_cost_callback``, so +spend logs, spend counters, budget reservation reconciliation and cache updates happen exactly as +they would in-process, just in this container. Events are handled in order per producer connection +(one per uvicorn worker); a slow pipeline fills the socket buffer and the producer's bounded queue, +which is the backpressure that triggers its fallback or drop policy. ``SIGTERM`` stops accepting +connections, half-closes every producer connection so the producers switch to their unavailable +policy, finishes the events already sent, then runs the proxy shutdown (which flushes the buffered +spend transactions). + +``DATABASE_URL`` is assembled from the same ``DATABASE_*`` inputs as the proxy container, and when +``LITELLM_PGBOUNCER_ENABLED`` is set it points at the PgBouncer that container already runs on the +pod's loopback, so the sidecar must see the same env as the proxy. Under ``IAM_TOKEN_DB_AUTH`` or +``AZURE_POSTGRESQL_AUTH`` that PgBouncer only accepts the token the proxy container minted, so the +sidecar goes to Postgres directly and mints its own. Works from any image that has ``litellm`` +installed: + + python -m litellm.proxy.collector [--address unix:///path.sock] +""" + +import asyncio +import logging +import os +import signal +import sys +from collections.abc import Awaitable, Callable, Mapping, Sequence +from pathlib import Path +from typing import Final + +from litellm._logging import verbose_logger, verbose_proxy_logger, verbose_router_logger +from litellm.proxy.db.db_url_settings import DatabaseURLSettings +from litellm.proxy.db.pgbouncer import ( + PgBouncerError, + PgBouncerSettings, + export_pooled_database_url, + pooled_database_url, +) +from litellm.proxy.spend_tracking.spend_event_producer import ( + COLLECTOR_JOB_ROLE, + AddressError, + CollectorAddress, + CollectorSettings, + TcpAddress, + UnixAddress, + parse_collector_address, +) + +MAX_EVENT_BYTES: Final = 64 * 1024 * 1024 + + +class SpendEventConsumer: + """Accepts producer connections and runs ``handler`` on every line each one sends, in order.""" + + def __init__(self, handler: Callable[[bytes], Awaitable[None]]) -> None: + self._handler = handler + self._open_connections: set[asyncio.StreamWriter] = set() # mutable-ok: live producer connections + self._idle = asyncio.Event() + self._idle.set() + self._received = 0 + self._handled = 0 + self._failed = 0 + + @property + def received(self) -> int: + return self._received + + @property + def handled(self) -> int: + return self._handled + + @property + def failed(self) -> int: + return self._failed + + async def serve(self, address: CollectorAddress) -> asyncio.Server: + match address: + case UnixAddress(path=path): + socket_path: Final = Path(path) + socket_path.parent.mkdir(parents=True, exist_ok=True) + socket_path.unlink(missing_ok=True) + return await asyncio.start_unix_server(self._on_connection, path=path, limit=MAX_EVENT_BYTES) + case TcpAddress(host=host, port=port): + return await asyncio.start_server(self._on_connection, host=host, port=port, limit=MAX_EVENT_BYTES) + + async def _on_connection(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + self._open_connections.add(writer) + self._idle.clear() + try: + while line := await reader.readline(): + if not line.endswith(b"\n"): + verbose_proxy_logger.error("collector: discarding truncated spend event (%d bytes)", len(line)) + break + self._received += 1 + await self._handle(line) + except (ConnectionError, asyncio.IncompleteReadError, asyncio.LimitOverrunError) as error: + verbose_proxy_logger.warning("collector: producer connection ended abnormally: %s", error) + finally: + writer.close() + self._open_connections.discard(writer) + if not self._open_connections: + self._idle.set() + + async def _handle(self, line: bytes) -> None: + try: + await self._handler(line) + self._handled += 1 + except Exception: # noqa: BLE001 # the cost pipeline raises anything; one bad event must not stop the sidecar + self._failed += 1 + verbose_proxy_logger.exception("collector: spend event failed") + + async def drain(self, timeout: float) -> int: + """Half-close every producer connection, then keep reading until each producer hangs up or ``timeout``. + + Returns how many producer connections were still open when the timeout hit. + """ + for writer in tuple(self._open_connections): + if writer.is_closing() or not writer.can_write_eof(): + continue + try: + writer.write_eof() + except (OSError, RuntimeError) as error: + verbose_proxy_logger.debug("collector: producer already gone before half-close: %s", error) + try: + await asyncio.wait_for(self._idle.wait(), timeout) + except TimeoutError: + pass + return len(self._open_connections) + + +def _install_stop_signals(loop: asyncio.AbstractEventLoop, stop: asyncio.Event) -> None: + for signum in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(signum, stop.set) + + +async def run_collector(address: CollectorAddress, drain_timeout: float) -> None: + from fastapi import FastAPI + + from litellm.proxy.hooks.proxy_track_cost_callback import run_spend_event + from litellm.proxy.proxy_server import proxy_startup_event + + stop: Final = asyncio.Event() + _install_stop_signals(asyncio.get_running_loop(), stop) + consumer: Final = SpendEventConsumer(handler=run_spend_event) + async with proxy_startup_event(FastAPI()): + server: Final = await consumer.serve(address) + verbose_proxy_logger.info("collector: listening on %s", address) + await stop.wait() + server.close() + still_open: Final = await consumer.drain(drain_timeout) + verbose_proxy_logger.info( + "collector: stopping. received=%d handled=%d failed=%d connections_cut=%d", + consumer.received, + consumer.handled, + consumer.failed, + still_open, + ) + + +def address_argument(argv: Sequence[str], default: str) -> str | AddressError: + match tuple(argv): + case (): + return default + case ("--address", value): + return value + case _: + return AddressError(f"usage: python -m litellm.proxy.collector [--address ADDRESS], got {tuple(argv)}") + + +def apply_log_level(litellm_log: str | None) -> None: + """Mirror the proxy's ``LITELLM_LOG`` handling: the sidecar has no CLI flags to turn logging on.""" + level: Final = logging.getLevelNamesMapping().get((litellm_log or "").upper()) + if level is None: + return + for logger in (verbose_logger, verbose_router_logger, verbose_proxy_logger): + logger.setLevel(level) + + +def pod_pgbouncer_database_url( + pgbouncer: PgBouncerSettings, environ: Mapping[str, str], *, token_auth: bool +) -> str | PgBouncerError | None: + """The proxy container's PgBouncer URL for ``environ["DATABASE_URL"]``, or None to connect to Postgres directly. + + Direct is the answer when PgBouncer is off, and also under token auth: that PgBouncer's auth file + only holds the token its own container minted, which this container cannot present. + """ + if not pgbouncer.enabled or token_auth: + return None + upstream_url: Final = environ.get("DATABASE_URL") + if upstream_url is None: + return PgBouncerError("LITELLM_PGBOUNCER_ENABLED is set but no DATABASE_URL could be assembled") + return pooled_database_url(upstream_url, pgbouncer) + + +def main(argv: Sequence[str]) -> None: + os.environ.setdefault("LITELLM_JOB_ROLE", COLLECTOR_JOB_ROLE) + apply_log_level(os.environ.get("LITELLM_LOG")) + database: Final = DatabaseURLSettings.from_env() + database.apply_to_env() + pooled: Final = pod_pgbouncer_database_url( + PgBouncerSettings(), + os.environ, + token_auth=database.iam_token_db_auth or database.azure_postgresql_auth, + ) + if isinstance(pooled, PgBouncerError): + sys.exit(f"LiteLLM collector: cannot use the pod's pgbouncer: {pooled.reason}") + if pooled is not None: + export_pooled_database_url(pooled) + settings: Final = CollectorSettings() + raw_address: Final = address_argument(argv, default=settings.address) + address: Final = raw_address if isinstance(raw_address, AddressError) else parse_collector_address(raw_address) + if isinstance(address, AddressError): + sys.exit(f"LiteLLM collector: {address.reason}") + asyncio.run(run_collector(address, drain_timeout=settings.drain_timeout_seconds)) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/litellm/proxy/db/pgbouncer.py b/litellm/proxy/db/pgbouncer.py index eac80f7a712..c9fbabd3585 100644 --- a/litellm/proxy/db/pgbouncer.py +++ b/litellm/proxy/db/pgbouncer.py @@ -280,6 +280,22 @@ def plan_pgbouncer( ) +def pooled_database_url(upstream_url: str, settings: PgBouncerSettings) -> str | PgBouncerError: + """The loopback URL of a PgBouncer another container in the pod already runs for ``upstream_url``. + + Only the container that started PgBouncer knows the pool user's password, so + this logs in as the upstream user, whom the auth file lists as well. + """ + plan: Final = plan_pgbouncer(upstream_url, settings, runtime_dir=Path("/nonexistent"), run_as_user=None) + if isinstance(plan, PgBouncerError): + return plan + password: Final = urllib.parse.urlsplit(upstream_url).password or "" + credentials: Final = f"{urllib.parse.quote(plan.upstream_user, safe='')}:{password}" + return urllib.parse.urlunsplit( + urllib.parse.urlsplit(plan.pooled_url)._replace(netloc=f"{credentials}@{PGBOUNCER_LISTEN_ADDR}:{settings.port}") + ) + + def _write_private(path: Path, content: str, run_as_user: str | None) -> None: with open(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "w", encoding="utf-8") as handle: handle.write(content) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index c4fba8ecf9e..00406ad436e 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -27,6 +27,16 @@ from litellm.proxy.db.db_spend_update_writer import ( get_llm_router, ) from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +from litellm.proxy.spend_tracking.spend_event import ( + ObjectMapping, + SpendEventBuildError, + SpendEventDecodeError, + build_spend_event, + decode_spend_event, + is_offloadable_success, + spend_event_callback_args, +) +from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer from litellm.proxy.spend_tracking.spend_log_error_logger import ( should_suppress_spend_log_tracebacks, spend_log_error, @@ -34,6 +44,7 @@ from litellm.proxy.spend_tracking.spend_log_error_logger import ( from litellm.proxy.spend_tracking.spend_tracking_utils import ( _sanitize_error_information_for_spend_logs, get_request_model_access_groups, + should_store_prompts_and_responses_in_spend_logs, ) from litellm.proxy.utils import ProxyUpdateSpend from litellm.types.utils import ( @@ -71,8 +82,43 @@ _CAPTURED_IDENTITY_CALL_TYPES: Final[frozenset[str]] = frozenset( class _ProxyDBLogger(CustomLogger): - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - await self._PROXY_track_cost_callback(kwargs, response_obj, start_time, end_time) + def __init__( + self, + spend_event_producer: SpendEventProducer | None = None, + *, + turn_off_message_logging: bool = False, + message_logging: bool = True, + ) -> None: + super().__init__(turn_off_message_logging=turn_off_message_logging, message_logging=message_logging) + self.spend_event_producer = spend_event_producer + + async def async_log_success_event( + self, kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + if self.spend_event_producer is None or not is_offloadable_success(response_obj): + await self._PROXY_track_cost_callback(kwargs, response_obj, start_time, end_time) + return + event: Final = build_spend_event( + kwargs, + response_obj, + start_time, + end_time, + store_bodies=should_store_prompts_and_responses_in_spend_logs(), + ) + if isinstance(event, SpendEventBuildError): + verbose_proxy_logger.warning("collector: tracking cost in-process, event not buildable: %s", event.reason) + await self._PROXY_track_cost_callback(kwargs, response_obj, start_time, end_time) + return + await self.spend_event_producer.publish(event) + + async def run_spend_event(self, line: bytes) -> None: + """Run the unchanged cost pipeline on a serialized spend event (sidecar consumer and in-process fallback).""" + event: Final = decode_spend_event(line) + if isinstance(event, SpendEventDecodeError): + verbose_proxy_logger.error("collector: discarding undecodable spend event: %s", event.reason) + return + args: Final = spend_event_callback_args(event) + await self._PROXY_track_cost_callback(args.kwargs, args.response_obj, args.start_time, args.end_time) async def async_post_call_failure_hook( self, @@ -503,6 +549,10 @@ def _write_spend_metadata_to_kwargs(kwargs: dict, metadata: dict) -> None: bucket[key] = value +async def run_spend_event(line: bytes) -> None: + await _ProxyDBLogger().run_spend_event(line) + + def _is_unbilled_interaction_response(completion_response: object) -> bool: from litellm.interactions.background_cost_polling import missing_usage_is_expected from litellm.types.interactions import InteractionsAPIResponse diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7b64f65c79b..0f24acb8bb4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -469,7 +469,7 @@ from litellm.proxy.hooks.model_max_budget_limiter import ( from litellm.proxy.hooks.prompt_injection_detection import ( _OPTIONAL_PromptInjectionDetection, ) -from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger +from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger, run_spend_event from litellm.proxy.image_endpoints.endpoints import router as image_router from litellm.proxy.list_api.common import ( PROBLEM_TYPE_BASE, @@ -592,6 +592,11 @@ from litellm.proxy.plugin_routes import ( from litellm.proxy.plugin_routes import ( router as plugin_router, ) +from litellm.proxy.spend_tracking.spend_event_producer import ( + CollectorSettings, + SpendEventProducer, + build_spend_event_producer, +) from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail try: @@ -943,6 +948,17 @@ def cleanup_router_config_variables(): heuristic_v1_tuning_baselines = None +async def flush_spend_counters_on_shutdown() -> None: + if prisma_client is None: + return + try: + await proxy_logging_obj.db_spend_update_writer.db_update_spend_transaction_handler( + prisma_client=prisma_client, n_retry_times=3, proxy_logging_obj=proxy_logging_obj + ) + except Exception as e: # noqa: BLE001 # shutdown must continue even if the commit fails + verbose_proxy_logger.exception("Error flushing spend counters on shutdown: %s", e) + + async def _flush_spend_logs_queue_on_shutdown() -> None: if prisma_client is None: return @@ -1378,6 +1394,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: except Exception as e: verbose_proxy_logger.error("Error stopping DB health watchdog task: %s", e) + await _drain_spend_event_producer_on_shutdown() + + await flush_spend_counters_on_shutdown() + await _flush_spend_logs_queue_on_shutdown() await proxy_config.stop_config_sync_subscriber() @@ -2457,16 +2477,27 @@ def load_from_azure_key_vault(use_azure_key_vault: bool = False): ) +spend_event_producer: SpendEventProducer | None = None + + def cost_tracking(): - global prisma_client + global prisma_client, spend_event_producer if prisma_client is not None: from litellm.integrations.shadow_eval_logger import ShadowEvalLogger - litellm.logging_callback_manager.add_litellm_callback(_ProxyDBLogger()) - litellm.logging_callback_manager.add_litellm_async_success_callback(_ProxyDBLogger()) + spend_event_producer = build_spend_event_producer(CollectorSettings(), fallback=run_spend_event) + litellm.logging_callback_manager.add_litellm_callback(_ProxyDBLogger(spend_event_producer)) + litellm.logging_callback_manager.add_litellm_async_success_callback(_ProxyDBLogger(spend_event_producer)) litellm.logging_callback_manager.add_litellm_callback(ShadowEvalLogger()) +async def _drain_spend_event_producer_on_shutdown() -> None: + if spend_event_producer is None: + return + await spend_event_producer.close(drain_timeout=CollectorSettings().drain_timeout_seconds) + verbose_proxy_logger.info("collector: producer drained on shutdown. stats=%s", spend_event_producer.stats()) + + # Bounds authoritative DB re-reads when enforcing a budget against a # stale-low spend counter: at most one DB read per counter per window. SPEND_DB_FLOOR_CACHE_TTL_SECONDS: Final = 5 diff --git a/litellm/proxy/spend_tracking/spend_event.py b/litellm/proxy/spend_tracking/spend_event.py new file mode 100644 index 00000000000..53f26346f85 --- /dev/null +++ b/litellm/proxy/spend_tracking/spend_event.py @@ -0,0 +1,418 @@ +"""Compact, typed success event handed from an inference worker to the collector. + +``build_spend_event`` runs on the inference worker right after ``Logging.async_success_handler`` +has built the ``standard_logging_object`` (so the cost is already known). It validates the success +callback's ``kwargs`` into the projection ``_PROXY_track_cost_callback`` and +``DBSpendUpdateWriter.update_database`` actually read: identities and metadata, timings, usage, the +standard logging payload without its prompt/response bodies, and the tool names. The request +messages, the raw ``proxy_server_request`` body and the full response travel only when spend logs +are configured to store prompts and responses. The cache key is the preset key the caching layer +already computed, never a fresh hash over the request body. + +``spend_event_callback_args`` rebuilds the ``(kwargs, response_obj, start_time, end_time)`` tuple +the existing cost pipeline consumes, so the sidecar runs the unchanged pipeline against the event. +""" + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from typing_extensions import NotRequired, ReadOnly, TypedDict + +import litellm +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.db.spend_log_tool_index import response_tool_call_names +from litellm.types.interactions import InteractionsAPIResponse +from litellm.types.utils import LiteLLMBatch, Usage + +SPEND_EVENT_VERSION: Final = 1 +CACHE_OFF_KEY: Final = "Cache OFF" + +ObjectMapping: TypeAlias = Mapping[str, object] + +_UNSERIALIZABLE_METADATA_KEYS: Final = frozenset({"user_api_key_auth", "litellm_parent_otel_span"}) +_STANDARD_LOGGING_BODY_KEYS: Final = frozenset({"messages", "response"}) +_STANDARD_LOGGING_DROPPED_KEYS: Final = frozenset({"model_parameters"}) +_NOT_OFFLOADED_RESPONSE_TYPES: Final = (LiteLLMBatch, InteractionsAPIResponse) + + +class _LitellmParams(TypedDict, total=False): + api_base: ReadOnly[str | None] + custom_llm_provider: ReadOnly[str | None] + litellm_call_id: ReadOnly[str | None] + user_api_key_end_user_id: ReadOnly[str | None] + metadata: ReadOnly[ObjectMapping | None] + litellm_metadata: ReadOnly[ObjectMapping | None] + proxy_server_request: ReadOnly[ObjectMapping | None] + preset_cache_key: ReadOnly[str | None] + + +class _DynamicParams(TypedDict, total=False): + turn_off_message_logging: ReadOnly[bool | None] + + +class _RequestBody(TypedDict, total=False): + tools: ReadOnly[Sequence[ObjectMapping] | None] + + +class _PassthroughPayload(TypedDict, total=False): + request_body: ReadOnly[_RequestBody | None] + + +class _ToolCallFunction(TypedDict): + name: ReadOnly[str] + arguments: ReadOnly[str] + + +class _ToolCall(TypedDict): + id: ReadOnly[str | None] + type: ReadOnly[Literal["function"]] + function: ReadOnly[_ToolCallFunction] + + +class _ToolCallMessage(TypedDict): + role: ReadOnly[Literal["assistant"]] + content: ReadOnly[None] + tool_calls: ReadOnly[Sequence[_ToolCall]] + + +class _ToolCallChoice(TypedDict): + index: ReadOnly[int] + finish_reason: ReadOnly[Literal["tool_calls"]] + message: ReadOnly[_ToolCallMessage] + + +class CompactResponse(TypedDict, total=False): + """What the spend pipeline reads off a response: its id, usage and which tools it called.""" + + id: ReadOnly[object] + model: ReadOnly[object] + usage: ReadOnly[object] + usage_info: ReadOnly[object] + status: ReadOnly[object] + background: ReadOnly[object] + choices: ReadOnly[Sequence[_ToolCallChoice]] + + +class _SuccessKwargs(TypedDict, total=False): + """The success callback's ``kwargs`` (``Logging.model_call_details``), validated and projected.""" + + litellm_call_id: ReadOnly[str | None] + call_type: ReadOnly[str | None] + model: ReadOnly[str | None] + custom_llm_provider: ReadOnly[str | None] + stream: ReadOnly[bool | None] + complete_streaming_response: ReadOnly[object] + cache_hit: ReadOnly[bool | None] + response_cost: ReadOnly[float | None] + completion_start_time: ReadOnly[datetime | None] + agent_id: ReadOnly[str | None] + litellm_trace_id: ReadOnly[str | None] + litellm_params: ReadOnly[_LitellmParams] + standard_logging_object: ReadOnly[ObjectMapping | None] + standard_callback_dynamic_params: ReadOnly[_DynamicParams | None] + combined_usage_object: ReadOnly[Usage | None] + realtime_tools: ReadOnly[Sequence[object] | None] + realtime_tool_calls: ReadOnly[Sequence[object] | None] + tools: ReadOnly[Sequence[ObjectMapping] | None] + passthrough_logging_payload: ReadOnly[_PassthroughPayload | None] + + +class _FunctionToolFunction(TypedDict): + name: ReadOnly[str] + + +class _FunctionTool(TypedDict): + type: ReadOnly[Literal["function"]] + function: ReadOnly[_FunctionToolFunction] + + +class SpendCallbackKwargs(TypedDict): + """The ``kwargs`` handed to ``_PROXY_track_cost_callback`` on the sidecar.""" + + litellm_call_id: ReadOnly[str | None] + call_type: ReadOnly[str | None] + model: ReadOnly[str | None] + custom_llm_provider: ReadOnly[str | None] + stream: ReadOnly[bool | None] + cache_hit: ReadOnly[bool | None] + response_cost: ReadOnly[float | None] + completion_start_time: ReadOnly[datetime | None] + agent_id: ReadOnly[str | None] + litellm_trace_id: ReadOnly[str | None] + litellm_params: ReadOnly[_LitellmParams] + standard_logging_object: ReadOnly[ObjectMapping | None] + standard_callback_dynamic_params: ReadOnly[_DynamicParams | None] + combined_usage_object: ReadOnly[Usage | None] + realtime_tools: ReadOnly[Sequence[object] | None] + realtime_tool_calls: ReadOnly[Sequence[object] | None] + tools: ReadOnly[Sequence[_FunctionTool] | None] + complete_streaming_response: NotRequired[ReadOnly[CompactResponse | None]] + + +_NO_LITELLM_PARAMS: Final[_LitellmParams] = {} +_SUCCESS_KWARGS: Final = TypeAdapter(_SuccessKwargs) +_OBJECT_MAPPING: Final = TypeAdapter(ObjectMapping) +_COMPACT_RESPONSE: Final = TypeAdapter(CompactResponse) + + +class SpendEvent(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + version: Literal[1] + litellm_call_id: str | None + call_type: str | None + model: str | None + custom_llm_provider: str | None + stream: bool | None + complete_streaming_response: bool + cache_hit: bool | None + response_cost: float | None + start_time: datetime + end_time: datetime + completion_start_time: datetime | None + agent_id: str | None + litellm_trace_id: str | None + litellm_params: _LitellmParams + standard_logging_object: ObjectMapping | None + standard_callback_dynamic_params: _DynamicParams | None + response: CompactResponse | None + combined_usage: ObjectMapping | None + realtime_tools: Sequence[object] | None + realtime_tool_calls: Sequence[object] | None + request_tool_names: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class SpendEventCallbackArgs: + kwargs: SpendCallbackKwargs + response_obj: CompactResponse | None + start_time: datetime + end_time: datetime + + +@dataclass(frozen=True, slots=True) +class SpendEventBuildError: + reason: str + + +@dataclass(frozen=True, slots=True) +class SpendEventDecodeError: + reason: str + + +def is_offloadable_success(response_obj: object) -> bool: + """Batch retrieves and interaction polls branch on the concrete response class, so they stay in-process.""" + return not isinstance(response_obj, _NOT_OFFLOADED_RESPONSE_TYPES) + + +def _json_fallback(value: object) -> str: + return str(value) + + +def _mapping_or_none(value: object) -> ObjectMapping | None: + try: + return _OBJECT_MAPPING.validate_python(value) + except ValidationError: + return None + + +def _drop_keys(mapping: ObjectMapping, keys: frozenset[str]) -> ObjectMapping: + return MappingProxyType({key: value for key, value in mapping.items() if key not in keys}) + + +def _budget_reservation(metadata: ObjectMapping) -> ObjectMapping | None: + """The admission-time reservation, wherever the request setup left it, so the sidecar can reconcile it.""" + direct: Final = _mapping_or_none(metadata.get("user_api_key_budget_reservation")) + if direct is not None: + return direct + auth: Final = metadata.get("user_api_key_auth") + if isinstance(auth, UserAPIKeyAuth): + return auth.budget_reservation + auth_mapping: Final = _mapping_or_none(auth) + return _mapping_or_none(auth_mapping.get("budget_reservation")) if auth_mapping is not None else None + + +def _metadata_for_event( + metadata: ObjectMapping | None, budget_reservation: ObjectMapping | None +) -> ObjectMapping | None: + if metadata is None: + return None + kept: Final = _drop_keys(metadata, _UNSERIALIZABLE_METADATA_KEYS) + if budget_reservation is None: + return kept + return MappingProxyType({**kept, "user_api_key_budget_reservation": budget_reservation}) + + +def _litellm_params_for_event( + litellm_params: _LitellmParams, cache_key: str | None, store_bodies: bool +) -> _LitellmParams: + metadata: Final = litellm_params.get("metadata") + litellm_metadata: Final = litellm_params.get("litellm_metadata") + budget_reservation: Final = next( + ( + reservation + for source in (litellm_metadata, metadata) + if source is not None and (reservation := _budget_reservation(source)) is not None + ), + None, + ) + projected: Final[_LitellmParams] = { + "api_base": litellm_params.get("api_base"), + "custom_llm_provider": litellm_params.get("custom_llm_provider"), + "litellm_call_id": litellm_params.get("litellm_call_id"), + "user_api_key_end_user_id": litellm_params.get("user_api_key_end_user_id"), + "metadata": _metadata_for_event(metadata, budget_reservation), + "litellm_metadata": _metadata_for_event(litellm_metadata, budget_reservation), + "proxy_server_request": litellm_params.get("proxy_server_request") if store_bodies else None, + "preset_cache_key": cache_key, + } + return projected + + +def _standard_logging_for_event(sl_object: ObjectMapping | None, store_bodies: bool) -> ObjectMapping | None: + if sl_object is None: + return None + dropped: Final = ( + _STANDARD_LOGGING_DROPPED_KEYS if store_bodies else _STANDARD_LOGGING_DROPPED_KEYS | _STANDARD_LOGGING_BODY_KEYS + ) + return _drop_keys(sl_object, dropped) + + +def _tool_call(name: str) -> _ToolCall: + tool_call: Final[_ToolCall] = {"id": None, "type": "function", "function": {"name": name, "arguments": "{}"}} + return tool_call + + +def _tool_call_choice(names: Sequence[str]) -> _ToolCallChoice: + choice: Final[_ToolCallChoice] = { + "index": 0, + "finish_reason": "tool_calls", + "message": {"role": "assistant", "content": None, "tool_calls": tuple(_tool_call(name) for name in names)}, + } + return choice + + +def _compact_response(response_obj: object) -> CompactResponse | None: + """Usage, identity and tool calls of the response, in chat-completions shape, without the content.""" + dumped: Final = response_obj.model_dump() if isinstance(response_obj, BaseModel) else _mapping_or_none(response_obj) + if dumped is None: + return None + scalars: Final = _COMPACT_RESPONSE.validate_python(_drop_keys(dumped, frozenset({"choices"}))) + tool_call_names: Final = response_tool_call_names(response_obj) + if not tool_call_names: + return scalars + with_tool_calls: Final[CompactResponse] = {**scalars, "choices": (_tool_call_choice(tool_call_names),)} + return with_tool_calls + + +def _tool_name(tool: ObjectMapping) -> str | None: + """Chat tools nest the name under ``function``; Anthropic and Responses API tools keep it at the top.""" + function: Final = _mapping_or_none(tool.get("function")) + name: Final = function.get("name") if function is not None else tool.get("name") + return name.strip() if isinstance(name, str) and name.strip() else None + + +def _request_tool_names(kwargs: _SuccessKwargs) -> tuple[str, ...]: + passthrough: Final = kwargs.get("passthrough_logging_payload") + request_body: Final = passthrough.get("request_body") if passthrough is not None else None + passthrough_tools: Final = request_body.get("tools") if request_body is not None else None + return tuple( + name + for source in (kwargs.get("tools"), passthrough_tools) + if source is not None + for tool in source + if (name := _tool_name(tool)) is not None + ) + + +def preset_spend_log_cache_key(litellm_params: _LitellmParams) -> str | None: + """The key the caching layer already stored in ``litellm_params``, or ``Cache OFF``; never hashes the body.""" + if litellm.cache is None: + return CACHE_OFF_KEY + return litellm_params.get("preset_cache_key") + + +def _function_tool(name: str) -> _FunctionTool: + tool: Final[_FunctionTool] = {"type": "function", "function": {"name": name}} + return tool + + +def build_spend_event( + raw_kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime, store_bodies: bool +) -> bytes | SpendEventBuildError: + """Validate the success callback's kwargs and serialize the event once, as a single JSON line.""" + try: + kwargs: Final = _SUCCESS_KWARGS.validate_python(raw_kwargs) + except ValidationError as error: + return SpendEventBuildError(reason=str(error)) + litellm_params: Final = kwargs.get("litellm_params", _NO_LITELLM_PARAMS) + sl_object: Final = kwargs.get("standard_logging_object") + cache_key: Final = preset_spend_log_cache_key(litellm_params) + response_cost: Final = sl_object.get("response_cost") if sl_object is not None else kwargs.get("response_cost") + combined_usage: Final = kwargs.get("combined_usage_object") + event: Final = SpendEvent( + version=SPEND_EVENT_VERSION, + litellm_call_id=kwargs.get("litellm_call_id"), + call_type=kwargs.get("call_type"), + model=kwargs.get("model"), + custom_llm_provider=kwargs.get("custom_llm_provider"), + stream=kwargs.get("stream"), + complete_streaming_response="complete_streaming_response" in kwargs, + cache_hit=kwargs.get("cache_hit"), + response_cost=response_cost if isinstance(response_cost, (int, float)) else None, + start_time=start_time, + end_time=end_time, + completion_start_time=kwargs.get("completion_start_time"), + agent_id=kwargs.get("agent_id"), + litellm_trace_id=kwargs.get("litellm_trace_id"), + litellm_params=_litellm_params_for_event(litellm_params, cache_key, store_bodies), + standard_logging_object=_standard_logging_for_event(sl_object, store_bodies), + standard_callback_dynamic_params=kwargs.get("standard_callback_dynamic_params"), + response=_compact_response(response_obj), + combined_usage=combined_usage.model_dump() if combined_usage is not None else None, + realtime_tools=kwargs.get("realtime_tools"), + realtime_tool_calls=kwargs.get("realtime_tool_calls"), + request_tool_names=_request_tool_names(kwargs), + ) + return event.model_dump_json(fallback=_json_fallback).encode() + b"\n" + + +def decode_spend_event(line: bytes) -> SpendEvent | SpendEventDecodeError: + try: + return SpendEvent.model_validate_json(line) + except ValidationError as error: + return SpendEventDecodeError(reason=str(error)) + + +def spend_event_callback_args(event: SpendEvent) -> SpendEventCallbackArgs: + """The ``(kwargs, response_obj, start_time, end_time)`` the in-process cost callback receives.""" + tools: Final = tuple(_function_tool(name) for name in event.request_tool_names) + kwargs: Final[SpendCallbackKwargs] = { + "litellm_call_id": event.litellm_call_id, + "call_type": event.call_type, + "model": event.model, + "custom_llm_provider": event.custom_llm_provider, + "stream": event.stream, + "cache_hit": event.cache_hit, + "response_cost": event.response_cost, + "completion_start_time": event.completion_start_time, + "agent_id": event.agent_id, + "litellm_trace_id": event.litellm_trace_id, + "litellm_params": event.litellm_params, + "standard_logging_object": event.standard_logging_object, + "standard_callback_dynamic_params": event.standard_callback_dynamic_params, + "combined_usage_object": Usage.model_validate(event.combined_usage) + if event.combined_usage is not None + else None, + "realtime_tools": event.realtime_tools, + "realtime_tool_calls": event.realtime_tool_calls, + "tools": tools or None, + } + if not event.complete_streaming_response: + return SpendEventCallbackArgs(kwargs, event.response, event.start_time, event.end_time) + streaming_kwargs: Final[SpendCallbackKwargs] = {**kwargs, "complete_streaming_response": event.response} + return SpendEventCallbackArgs(streaming_kwargs, event.response, event.start_time, event.end_time) diff --git a/litellm/proxy/spend_tracking/spend_event_producer.py b/litellm/proxy/spend_tracking/spend_event_producer.py new file mode 100644 index 00000000000..20c7f177af0 --- /dev/null +++ b/litellm/proxy/spend_tracking/spend_event_producer.py @@ -0,0 +1,338 @@ +"""Fire-and-forget push of serialized spend events from an inference worker to the pod-local sidecar. + +``LITELLM_COLLECTOR_ENABLED=true`` turns the push on in the gateway; the sidecar process sets +``LITELLM_JOB_ROLE=collector`` and always runs the pipeline in-process. Events queue in a bounded +in-memory buffer that a single writer task flushes over a unix socket or loopback TCP connection. +When the sidecar is unreachable, the buffer is full, or the connection breaks mid-write, each affected +event follows ``LITELLM_COLLECTOR_ON_UNAVAILABLE``: ``fallback`` runs the existing cost pipeline in +the worker, ``drop`` counts it and moves on. Transitions are logged with the counters, so a sidecar +outage is visible without scraping anything. + +Delivery is at-most-once: a sidecar crash loses the events the kernel already took from its socket. +A sidecar that stops gracefully half-closes each connection first (EOF towards the producer) and +keeps reading until the producer hangs up, so the producer switches to the unavailable policy without +losing the events in flight. A write that fails part-way follows the unavailable policy without double +counting: ``drain()`` only fails while part of the line is still buffered in this process, so the +sidecar can at most have read a truncated line, which it discards. When the gateway itself stops with +the writer stuck mid-send, only an event whose bytes are still in the producer's write buffer follows +the unavailable policy; the connection is aborted first so the sidecar discards the truncated line +instead of also counting it. Events from one uvicorn worker are handled in the order it produced them; +events from different workers interleave, exactly like the in-process callbacks do today. +""" + +import asyncio +import ipaddress +import time +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Final, Literal, TypeAlias +from urllib.parse import urlsplit + +from pydantic import AliasChoices, Field +from pydantic_settings import BaseSettings, SettingsConfigDict + +from litellm._logging import verbose_proxy_logger + +COLLECTOR_ENV_PREFIX: Final = "LITELLM_COLLECTOR_" +COLLECTOR_JOB_ROLE: Final = "collector" +DEFAULT_COLLECTOR_ADDRESS: Final = "unix:///var/run/litellm/collector.sock" +RECONNECT_BACKOFF_SECONDS: Final = 1.0 +DROP_LOG_EVERY: Final = 1000 + +UnavailablePolicy: TypeAlias = Literal["fallback", "drop"] +PublishOutcome: TypeAlias = Literal["queued", "fallback", "dropped"] + + +class CollectorSettings(BaseSettings): + """``LITELLM_COLLECTOR_*`` env vars, shared by the gateway producer and the sidecar consumer.""" + + model_config = SettingsConfigDict( + env_prefix=COLLECTOR_ENV_PREFIX, case_sensitive=False, extra="ignore", frozen=True, populate_by_name=True + ) + + enabled: bool = False + address: str = DEFAULT_COLLECTOR_ADDRESS + buffer_size: int = Field(default=1000, ge=1) + on_unavailable: UnavailablePolicy = "fallback" + drain_timeout_seconds: float = Field(default=10.0, gt=0) + connect_timeout_seconds: float = Field(default=1.0, gt=0) + job_role: str | None = Field(default=None, validation_alias=AliasChoices("LITELLM_JOB_ROLE")) + + @property + def produces(self) -> bool: + return self.enabled and self.job_role != COLLECTOR_JOB_ROLE + + +@dataclass(frozen=True, slots=True) +class UnixAddress: + path: str + + +@dataclass(frozen=True, slots=True) +class TcpAddress: + host: str + port: int + + +@dataclass(frozen=True, slots=True) +class AddressError: + reason: str + + +CollectorAddress: TypeAlias = UnixAddress | TcpAddress + + +def _is_loopback(host: str) -> bool: + try: + return ipaddress.ip_address(host).is_loopback + except ValueError: + return host == "localhost" + + +def parse_collector_address(address: str) -> CollectorAddress | AddressError: + """``unix:///path/to.sock`` or ``tcp://127.0.0.1:port``; the socket carries unauthenticated spend events.""" + parsed: Final = urlsplit(address) + if parsed.scheme == "unix" and parsed.path: + return UnixAddress(path=parsed.path) + if parsed.scheme == "tcp" and parsed.hostname and parsed.port is not None: + if not _is_loopback(parsed.hostname): + return AddressError(reason=f"tcp collector address must be a loopback host, got {address!r}") + return TcpAddress(host=parsed.hostname, port=parsed.port) + return AddressError(reason=f"expected unix:///path or tcp://127.0.0.1:port, got {address!r}") + + +async def open_collector_connection( + address: CollectorAddress, timeout: float +) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]: + match address: + case UnixAddress(path=path): + return await asyncio.wait_for(asyncio.open_unix_connection(path), timeout) + case TcpAddress(host=host, port=port): + return await asyncio.wait_for(asyncio.open_connection(host, port), timeout) + + +def build_spend_event_producer( + settings: CollectorSettings, fallback: Callable[[bytes], Awaitable[None]] +) -> "SpendEventProducer | None": + """The gateway producer for these settings, or ``None`` when the pipeline stays in-process.""" + if not settings.produces: + return None + address: Final = parse_collector_address(settings.address) + if isinstance(address, AddressError): + verbose_proxy_logger.error("collector: %s; running the spend pipeline in-process", address.reason) + return None + verbose_proxy_logger.info( + "collector: offloading spend tracking to %s (buffer=%d, on_unavailable=%s)", + settings.address, + settings.buffer_size, + settings.on_unavailable, + ) + return SpendEventProducer( + address=address, + on_unavailable=settings.on_unavailable, + buffer_size=settings.buffer_size, + connect_timeout=settings.connect_timeout_seconds, + fallback=fallback, + ) + + +@dataclass(frozen=True, slots=True) +class _Connection: + reader: asyncio.StreamReader + writer: asyncio.StreamWriter + + @property + def alive(self) -> bool: + return not self.writer.is_closing() and not self.reader.at_eof() + + +@dataclass(frozen=True, slots=True) +class SpendEventProducerStats: + queued: int + sent: int + fallback: int + dropped: int + connected: bool + + +class SpendEventProducer: + """Bounded buffer plus one writer task per process; see the module docstring for the contract.""" + + def __init__( + self, + address: CollectorAddress, + on_unavailable: UnavailablePolicy, + buffer_size: int, + connect_timeout: float, + fallback: Callable[[bytes], Awaitable[None]], + clock: Callable[[], float] = time.monotonic, + open_connection: Callable[ + [CollectorAddress, float], Awaitable[tuple[asyncio.StreamReader, asyncio.StreamWriter]] + ] = open_collector_connection, + ) -> None: + self._address = address + self._on_unavailable = on_unavailable + self._buffer_size = buffer_size + self._connect_timeout = connect_timeout + self._fallback = fallback + self._clock = clock + self._open_connection = open_connection + self._queue: asyncio.Queue[bytes] | None = None + self._writer_task: asyncio.Task[None] | None = None + self._connection: _Connection | None = None + self._in_flight: bytes | None = None + self._closing = False + self._next_connect_at = 0.0 + self._queued = 0 + self._sent = 0 + self._fallback_count = 0 + self._dropped = 0 + + def stats(self) -> SpendEventProducerStats: + return SpendEventProducerStats( + queued=self._queued, + sent=self._sent, + fallback=self._fallback_count, + dropped=self._dropped, + connected=self._connection is not None, + ) + + async def publish(self, line: bytes) -> PublishOutcome: + """Hand one serialized event to the writer task, or apply the unavailable policy right away.""" + if self._closing or self._clock() < self._next_connect_at: + return await self._unavailable(line, "sidecar unreachable") + queue: Final = self._ensure_writer() + try: + queue.put_nowait(line) + except asyncio.QueueFull: + return await self._unavailable(line, "buffer full") + self._queued += 1 + return "queued" + + async def close(self, drain_timeout: float) -> None: + """Flush the buffer for up to ``drain_timeout`` seconds, then apply the unavailable policy to the rest.""" + self._closing = True + queue: Final = self._queue + task: Final = self._writer_task + if queue is None or task is None: + return + try: + await asyncio.wait_for(queue.join(), drain_timeout) + except asyncio.TimeoutError: + verbose_proxy_logger.warning( + "collector: %s events still buffered after %.1fs drain timeout", queue.qsize(), drain_timeout + ) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + unsent: Final = self._take_unsent() + await self._disconnect() + if unsent is not None: + await self._unavailable(unsent, "shutdown") + while not queue.empty(): + await self._unavailable(queue.get_nowait(), "shutdown") + + def _take_unsent(self) -> bytes | None: + """The in-flight event if any of its bytes never left this process, aborting the half-written connection.""" + in_flight: Final = self._in_flight + self._in_flight = None + connection: Final = self._connection + if in_flight is None: + return None + if connection is None: + return in_flight + if connection.writer.transport.get_write_buffer_size() == 0: + return None + connection.writer.transport.abort() + return in_flight + + def _ensure_writer(self) -> asyncio.Queue[bytes]: + if self._queue is None: + self._queue = asyncio.Queue(maxsize=self._buffer_size) + if self._writer_task is None or self._writer_task.done(): + self._writer_task = asyncio.get_running_loop().create_task(self._run_writer(self._queue)) + return self._queue + + async def _run_writer(self, queue: asyncio.Queue[bytes]) -> None: + while True: + line = await queue.get() + try: + await self._send(line) + finally: + queue.task_done() + + async def _send(self, line: bytes) -> None: + self._in_flight = line + connection: Final = await self._connect() + if connection is None: + self._in_flight = None + await self._unavailable(line, "sidecar unreachable") + return + try: + connection.writer.write(line) + await connection.writer.drain() + except (ConnectionError, OSError, RuntimeError) as error: # uvloop: RuntimeError on a closed transport + self._in_flight = None + await self._disconnect() + self._next_connect_at = self._clock() + RECONNECT_BACKOFF_SECONDS + await self._unavailable(line, f"write failed: {error}") + return + self._in_flight = None + self._sent += 1 + + async def _connect(self) -> _Connection | None: + if self._connection is not None and self._connection.alive: + return self._connection + await self._disconnect() + if self._clock() < self._next_connect_at: + return None + try: + reader, writer = await self._open_connection(self._address, self._connect_timeout) + except (ConnectionError, OSError, asyncio.TimeoutError) as error: + self._next_connect_at = self._clock() + RECONNECT_BACKOFF_SECONDS + verbose_proxy_logger.warning( + "collector: cannot reach %s (%s); applying %s policy for %.0fs. stats=%s", + self._address, + error, + self._on_unavailable, + RECONNECT_BACKOFF_SECONDS, + self.stats(), + ) + return None + self._connection = _Connection(reader=reader, writer=writer) + verbose_proxy_logger.info("collector: connected to %s. stats=%s", self._address, self.stats()) + return self._connection + + async def _disconnect(self) -> None: + connection: Final = self._connection + self._connection = None + if connection is None: + return + connection.writer.close() + try: + await connection.writer.wait_closed() + except (ConnectionError, OSError): + pass + + async def _unavailable(self, line: bytes, reason: str) -> PublishOutcome: + if self._on_unavailable == "fallback": + self._fallback_count += 1 + fallback: Final = asyncio.ensure_future(self._run_fallback(line, reason)) + try: + await asyncio.shield(fallback) + except asyncio.CancelledError: + await fallback + raise + return "fallback" + self._dropped += 1 + if self._dropped % DROP_LOG_EVERY == 1: + verbose_proxy_logger.warning("collector: dropping spend event (%s). stats=%s", reason, self.stats()) + return "dropped" + + async def _run_fallback(self, line: bytes, reason: str) -> None: + try: + await self._fallback(line) + except Exception: # noqa: BLE001 # one failing event must not kill the writer task + verbose_proxy_logger.exception("collector: in-process fallback failed (%s)", reason) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index dad9a2a8777..f0f38358cf0 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -554,10 +554,12 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs additional_usage_values["cache_creation_input_tokens"] = cache_write_tokens clean_metadata["additional_usage_values"] = additional_usage_values - if litellm.cache is not None: - cache_key = litellm.cache.get_cache_key(**kwargs) - else: + if litellm.cache is None: cache_key = "Cache OFF" + elif litellm_params.get("preset_cache_key") is not None: + cache_key = litellm_params["preset_cache_key"] + else: + cache_key = litellm.cache.get_cache_key(**kwargs) if cache_hit is True: import time @@ -864,7 +866,7 @@ def _get_messages_for_spend_logs_payload( standard_logging_payload: StandardLoggingPayload | None, metadata: dict | None = None, ) -> str: - if _should_store_prompts_and_responses_in_spend_logs(): + if should_store_prompts_and_responses_in_spend_logs(): if standard_logging_payload is not None: call_type: Final = standard_logging_payload.get("call_type", "") if call_type == "_arealtime": @@ -1114,7 +1116,7 @@ def _sanitize_guardrail_information_for_spend_logs( here to match OTEL's defensive read pattern; otherwise iteration would yield the dict's keys and crash the whole spend-log write. """ - if guardrail_information is None or _should_store_prompts_and_responses_in_spend_logs(): + if guardrail_information is None or should_store_prompts_and_responses_in_spend_logs(): return guardrail_information entries: Final = [guardrail_information] if isinstance(guardrail_information, dict) else guardrail_information return [_redact_prompt_fields_in_guardrail_entry(entry) for entry in entries if isinstance(entry, dict)] @@ -1186,7 +1188,7 @@ def _sanitize_error_information_for_spend_logs( sanitized = cast(dict, {**error_information}) - if not _should_store_prompts_and_responses_in_spend_logs(): + if not should_store_prompts_and_responses_in_spend_logs(): for field in ("error_message", "traceback"): value = sanitized.get(field) if isinstance(value, str): @@ -1263,11 +1265,11 @@ def _get_proxy_server_request_for_spend_logs_payload( kwargs: dict | None = None, ) -> str: """ - Only store if _should_store_prompts_and_responses_in_spend_logs() is True + Only store if should_store_prompts_and_responses_in_spend_logs() is True If turn_off_message_logging is enabled, redact messages in the request body. """ - if _should_store_prompts_and_responses_in_spend_logs(): + if should_store_prompts_and_responses_in_spend_logs(): _proxy_server_request: Final = cast(dict | None, litellm_params.get("proxy_server_request", EMPTY_MAPPING)) if _proxy_server_request is not None: _request_body = _proxy_server_request.get("body", EMPTY_MAPPING) or EMPTY_MAPPING @@ -1317,7 +1319,7 @@ def _get_vector_store_request_for_spend_logs_payload( """ If user does not want to store prompts and responses, then remove the content from the vector store request metadata """ - if _should_store_prompts_and_responses_in_spend_logs(): + if should_store_prompts_and_responses_in_spend_logs(): return vector_store_request_metadata # if user does not want to store prompts and responses, then remove the content from the vector store request metadata @@ -1341,7 +1343,7 @@ def _get_response_for_spend_logs_payload( ) -> str: if payload is None: return "{}" - if _should_store_prompts_and_responses_in_spend_logs(): + if should_store_prompts_and_responses_in_spend_logs(): response_obj: object = payload.get("response") if response_obj is None: return "{}" @@ -1389,7 +1391,7 @@ def _get_response_for_spend_logs_payload( return "{}" -def _should_store_prompts_and_responses_in_spend_logs() -> bool: +def should_store_prompts_and_responses_in_spend_logs() -> bool: from litellm.proxy.proxy_server import general_settings from litellm.secret_managers.main import get_secret_bool diff --git a/litellm/utils.py b/litellm/utils.py index 69ee730ff30..aced4c9b312 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1843,6 +1843,9 @@ def client(original_function): elif _caching_handler_response.embedding_all_elements_cache_hit is True: return _caching_handler_response.final_embedding_cached_response + if _llm_caching_handler.preset_cache_key is not None: + logging_obj.litellm_params["preset_cache_key"] = _llm_caching_handler.preset_cache_key + # CHECK MAX TOKENS if ( kwargs.get("max_tokens", None) is not None diff --git a/terraform/litellm/README.md b/terraform/litellm/README.md index d4b40741052..7cc6e4c08ba 100644 --- a/terraform/litellm/README.md +++ b/terraform/litellm/README.md @@ -183,6 +183,7 @@ only where the underlying cloud forces it. | Extra secret-backed env | `gateway_extra_secrets`, `backend_extra_secrets` (ARNs) | `gateway_extra_secrets`, `backend_extra_secrets` (resource IDs) | | Uvicorn `--workers` on gateway | `gateway_num_workers` | `gateway_num_workers` | | OpenTelemetry v2 (opt-in) | `otel_endpoint`, `otel_exporter`, `otel_environment_name`, `otel_capture_message_content`, `otel_headers_secret_arn` | `otel_endpoint`, `otel_exporter`, `otel_environment_name`, `otel_capture_message_content`, `otel_headers_secret` | +| Collector sidecar (opt-in) | `collector_enabled`, `collector_port`, `collector_cpu`, `collector_memory`, `collector_buffer_size`, `collector_on_unavailable`, `collector_drain_timeout_seconds` | same names; `collector_cpu` / `collector_memory` take Cloud Run strings | Each module stamps its own stack-identity tag (`litellm:stack` on AWS, `litellm-stack` on GCP — GCP label keys forbid colons) plus diff --git a/terraform/litellm/aws/README.md b/terraform/litellm/aws/README.md index 5ca45944c54..67f1270d0c2 100644 --- a/terraform/litellm/aws/README.md +++ b/terraform/litellm/aws/README.md @@ -345,6 +345,39 @@ ten tasks handle 4,200,000,000 tokens in a minute, `tokens / 60` is `ceil(10 * 7000000 / 6000000) = 12`. Container Insights must be enabled on the cluster for `RunningTaskCount` to exist +### Collector sidecar + +`collector_enabled = true` adds a second container to the gateway task +that runs `python -m litellm.proxy.collector` from the gateway image, and sets +`LITELLM_COLLECTOR_ENABLED=true` on the gateway so its uvicorn workers +ship spend events (SpendLogs writes, key/team/user spend updates, budget +alerts) to the sidecar instead of running that pipeline in the request +path. This is the Terraform counterpart of helm's `gateway.collector`. +The default (`false`) leaves the task definition exactly as before. + +Fargate tasks share one network namespace, so the sidecar listens on +loopback TCP (`tcp://127.0.0.1:${collector_port}`, default 4010) instead +of the Unix socket helm uses; the proxy rejects any non-loopback address. +The sidecar gets the same database, Redis, master-key, license, proxy +config, and `gateway_extra_env` / `gateway_extra_secrets` values as the +gateway container, runs with `LITELLM_JOB_ROLE=collector`, and is +non-essential with an ECS restart policy, so a sidecar crash restarts it in +place while the gateway falls back to in-process spend tracking. + +```hcl +collector_enabled = true +# collector_cpu = 512 # carved out of gateway_cpu +# collector_memory = 2048 # MiB, carved out of gateway_memory +# collector_buffer_size = 1000 +# collector_on_unavailable = "fallback" # or "drop" +# collector_drain_timeout_seconds = 10 +``` + +Both sidecar reservations must leave room for the gateway container inside +`gateway_cpu` / `gateway_memory` (the plan fails otherwise). Service +autoscaling keeps tracking the whole task's CPU and memory, sidecar +included + ## Tenant deployment Every resource the stack creates is named `${tenant}-litellm-${env}` (or diff --git a/terraform/litellm/aws/ecs.tf b/terraform/litellm/aws/ecs.tf index 7176921e8fa..2b235c2bad5 100644 --- a/terraform/litellm/aws/ecs.tf +++ b/terraform/litellm/aws/ecs.tf @@ -278,6 +278,62 @@ locals { "${local.proxy_config_fetch_cmd} && ${local.backend_launch_cmd}" ] } : {} + + collector_address = "tcp://127.0.0.1:${var.collector_port}" + collector_env = var.collector_enabled ? [ + { name = "LITELLM_COLLECTOR_ENABLED", value = "true" }, + { name = "LITELLM_COLLECTOR_ADDRESS", value = local.collector_address }, + { name = "LITELLM_COLLECTOR_BUFFER_SIZE", value = tostring(var.collector_buffer_size) }, + { name = "LITELLM_COLLECTOR_ON_UNAVAILABLE", value = var.collector_on_unavailable }, + { name = "LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS", value = tostring(var.collector_drain_timeout_seconds) }, + ] : [] + + gateway_environment = concat( + local.shared_env, + local.gateway_otel_env, + local.billing_metrics_env, + local.gateway_extra_env_list, + local.proxy_config_env, + local.metrics_env, + local.gateway_pool_env, + local.collector_env, + ) + + collector_launch_cmd = "exec python -m litellm.proxy.collector" + collector_command = [ + local.proxy_config_enabled ? "${local.proxy_config_fetch_cmd} && ${local.collector_launch_cmd}" : local.collector_launch_cmd + ] + + collector_container = var.collector_enabled ? [{ + name = "collector" + image = var.gateway_image + essential = false + cpu = var.collector_cpu + memory = var.collector_memory + + restartPolicy = { enabled = true } + + entryPoint = ["sh", "-c"] + command = local.collector_command + environment = concat( + local.shared_env, + local.gateway_extra_env_list, + local.proxy_config_env, + local.gateway_pool_env, + local.collector_env, + [{ name = "LITELLM_JOB_ROLE", value = "collector" }], + ) + secrets = concat(local.shared_secrets, local.gateway_extra_secrets_list) + + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.gateway.name + awslogs-region = var.region + awslogs-stream-prefix = "collector" + } + } + }] : [] } # ---------- Gateway ---------- @@ -309,6 +365,16 @@ resource "aws_ecs_task_definition" "gateway" { condition = !var.gateway_connection_pool_enabled || local.database_enabled error_message = "gateway_connection_pool_enabled needs a database: set create_database = true or pass database_url." } + + precondition { + condition = !var.collector_enabled || (var.collector_cpu < var.gateway_cpu && var.collector_memory < var.gateway_memory) + error_message = "collector_cpu and collector_memory are carved out of gateway_cpu / gateway_memory and must leave room for the gateway container." + } + + precondition { + condition = !var.collector_enabled || var.gateway_metrics_port == null || var.collector_port != var.gateway_metrics_port + error_message = "collector_port and gateway_metrics_port must differ: both sidecars bind loopback in the same task." + } } family = "${local.name}-gateway" @@ -327,17 +393,9 @@ resource "aws_ecs_task_definition" "gateway" { essential = true portMappings = [{ containerPort = 4000, protocol = "tcp" }] - environment = concat( - local.shared_env, - local.gateway_otel_env, - local.billing_metrics_env, - local.gateway_extra_env_list, - local.proxy_config_env, - local.metrics_env, - local.gateway_pool_env, - ) - secrets = concat(local.shared_secrets, local.gateway_extra_secrets_list) - mountPoints = local.metrics_mount_points + environment = local.gateway_environment + secrets = concat(local.shared_secrets, local.gateway_extra_secrets_list) + mountPoints = local.metrics_mount_points # Container-level healthCheck intentionally omitted — the wolfi # runtime image doesn't ship curl/wget. The ALB target group polls @@ -354,7 +412,7 @@ resource "aws_ecs_task_definition" "gateway" { }, local.gateway_proxy_overrides, ) - ], local.gateway_metrics_container)) + ], local.gateway_metrics_container, local.collector_container)) dynamic "volume" { for_each = local.metrics_enabled ? [1] : [] diff --git a/terraform/litellm/aws/tests/collector.tftest.hcl b/terraform/litellm/aws/tests/collector.tftest.hcl new file mode 100644 index 00000000000..1465130232f --- /dev/null +++ b/terraform/litellm/aws/tests/collector.tftest.hcl @@ -0,0 +1,144 @@ +# Plan-only coverage for the opt-in collector sidecar in the gateway task. +# The rendered container_definitions JSON is unknown at plan time (it embeds +# Aurora/ElastiCache endpoints and secret ARNs), so the assertions target the +# locals it is built from. Run from terraform/litellm/aws with `terraform test`. + +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } +} +mock_provider "random" {} + +variables { + region = "us-east-1" + tenant = "acme" + env = "test" + allow_plaintext_alb = true + azs = ["us-east-1a", "us-east-1b"] +} + +run "disabled_by_default_leaves_the_task_untouched" { + command = plan + + assert { + condition = length(local.collector_container) == 0 + error_message = "The gateway task must stay single-container unless collector_enabled is set." + } + + assert { + condition = !anytrue([for e in local.gateway_environment : startswith(e.name, "LITELLM_COLLECTOR_")]) + error_message = "No LITELLM_COLLECTOR_* env may reach the gateway while the sidecar is disabled." + } +} + +run "enabled_adds_a_sidecar_that_shares_the_gateway_transport" { + command = plan + + variables { + collector_enabled = true + collector_port = 4321 + collector_buffer_size = 250 + collector_on_unavailable = "drop" + gateway_extra_env = { OPENAI_API_BASE = "https://example.invalid" } + gateway_extra_secrets = { OPENAI_API_KEY = "arn:aws:secretsmanager:us-east-1:111122223333:secret:openai-AbCdEf" } + } + + assert { + condition = length(local.collector_container) == 1 && local.collector_container[0].name == "collector" + error_message = "Enabling the sidecar must add exactly one collector container." + } + + assert { + condition = alltrue([ + for env in [local.gateway_environment, local.collector_container[0].environment] : ( + { for e in env : e.name => e.value }["LITELLM_COLLECTOR_ENABLED"] == "true" && + { for e in env : e.name => e.value }["LITELLM_COLLECTOR_ADDRESS"] == "tcp://127.0.0.1:4321" && + { for e in env : e.name => e.value }["LITELLM_COLLECTOR_BUFFER_SIZE"] == "250" && + { for e in env : e.name => e.value }["LITELLM_COLLECTOR_ON_UNAVAILABLE"] == "drop" && + { for e in env : e.name => e.value }["LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS"] == "10" + ) + ]) + error_message = "Gateway and sidecar must agree on the loopback address and the collector knobs." + } + + assert { + condition = ( + local.collector_container[0].image == var.gateway_image && + local.collector_container[0].entryPoint == ["sh", "-c"] && + local.collector_container[0].command == ["exec python -m litellm.proxy.collector"] && + local.collector_container[0].essential == false && + local.collector_container[0].restartPolicy.enabled == true && + { for e in local.collector_container[0].environment : e.name => e.value }["LITELLM_JOB_ROLE"] == "collector" + ) + error_message = "The sidecar must run litellm.proxy.collector from the gateway image as a restartable, non-essential collector." + } + + assert { + condition = ( + { for e in local.collector_container[0].environment : e.name => e.value }["OPENAI_API_BASE"] == "https://example.invalid" && + contains([for e in local.collector_container[0].environment : e.name], "DATABASE_HOST") && + contains([for e in local.collector_container[0].environment : e.name], "REDIS_HOST") && + contains([for s in local.collector_container[0].secrets : s.name], "LITELLM_MASTER_KEY") && + contains([for s in local.collector_container[0].secrets : s.name], "OPENAI_API_KEY") + ) + error_message = "The sidecar must receive the gateway's database, Redis, and shared secrets plus gateway_extra_env / gateway_extra_secrets." + } + + assert { + condition = !contains(keys(local.collector_container[0]), "portMappings") + error_message = "The sidecar must not expose a port to the task's load balancer." + } + + assert { + condition = local.collector_container[0].cpu == 512 && local.collector_container[0].memory == 2048 + error_message = "The sidecar defaults must mirror helm's collector resources (500m / 2Gi)." + } +} + +run "proxy_config_is_fetched_by_the_sidecar_too" { + command = plan + + variables { + collector_enabled = true + proxy_config = { model_list = [] } + } + + assert { + condition = ( + startswith(local.collector_container[0].command[0], local.proxy_config_fetch_cmd) && + endswith(local.collector_container[0].command[0], "exec python -m litellm.proxy.collector") && + contains([for e in local.collector_container[0].environment : e.name], "CONFIG_FILE_PATH") + ) + error_message = "The sidecar must pull the proxy config from S3 before starting, like the gateway does." + } +} + +run "sidecar_must_leave_room_for_the_gateway" { + command = plan + + variables { + collector_enabled = true + collector_cpu = 1024 + } + + expect_failures = [ + aws_ecs_task_definition.gateway, + ] +} + +run "sidecars_must_not_share_a_loopback_port" { + command = plan + + variables { + collector_enabled = true + collector_port = 4001 + gateway_metrics_port = 4001 + } + + expect_failures = [ + aws_ecs_task_definition.gateway, + ] +} diff --git a/terraform/litellm/aws/variables.tf b/terraform/litellm/aws/variables.tf index f57bb1ab7f6..580a0cc657a 100644 --- a/terraform/litellm/aws/variables.tf +++ b/terraform/litellm/aws/variables.tf @@ -808,3 +808,73 @@ variable "billing_metrics_ca_cert_pem" { default = "" sensitive = true } + +# ---------- Collector sidecar ---------- +# +# Opt-in offload of spend tracking from the gateway's uvicorn workers to a +# `python -m litellm.proxy.collector` sidecar in the same Fargate task (helm's +# `gateway.collector`). Fargate awsvpc tasks share one network namespace, +# so the sidecar listens on loopback TCP. Disabled (the default) adds nothing +# to the task definition. + +variable "collector_enabled" { + description = "Run the collector sidecar next to the gateway container and have the gateway ship spend events to it (sets LITELLM_COLLECTOR_ENABLED=true on both). Autoscaling still targets the whole task's CPU/memory, sidecar included." + type = bool + default = false +} + +variable "collector_port" { + description = "Loopback TCP port the sidecar listens on (LITELLM_COLLECTOR_ADDRESS=tcp://127.0.0.1:)." + type = number + default = 4010 + + validation { + condition = var.collector_port >= 1024 && var.collector_port <= 65535 && var.collector_port != 4000 + error_message = "collector_port must be in 1024-65535 and not 4000." + } +} + +variable "collector_cpu" { + description = "CPU units reserved for the sidecar container, carved out of gateway_cpu. Matches helm's collector.resources.requests.cpu (500m)." + type = number + default = 512 +} + +variable "collector_memory" { + description = "Hard memory limit (MiB) for the sidecar container, carved out of gateway_memory. Matches helm's collector.resources.limits.memory (2Gi)." + type = number + default = 2048 +} + +variable "collector_buffer_size" { + description = "Per-worker in-memory queue of spend events waiting to be shipped to the sidecar (LITELLM_COLLECTOR_BUFFER_SIZE)." + type = number + default = 1000 + + validation { + condition = var.collector_buffer_size >= 1 + error_message = "collector_buffer_size must be >= 1." + } +} + +variable "collector_on_unavailable" { + description = "What the gateway does with spend events when the sidecar is unreachable or the buffer is full (LITELLM_COLLECTOR_ON_UNAVAILABLE): `fallback` runs the pipeline in-process, `drop` discards them." + type = string + default = "fallback" + + validation { + condition = contains(["fallback", "drop"], var.collector_on_unavailable) + error_message = "collector_on_unavailable must be one of: fallback, drop." + } +} + +variable "collector_drain_timeout_seconds" { + description = "Seconds a gateway worker waits on shutdown for its buffered spend events to reach the sidecar (LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS)." + type = number + default = 10 + + validation { + condition = var.collector_drain_timeout_seconds > 0 + error_message = "collector_drain_timeout_seconds must be > 0." + } +} diff --git a/terraform/litellm/gcp/README.md b/terraform/litellm/gcp/README.md index 9c71d3e15b7..23cfe9afc24 100644 --- a/terraform/litellm/gcp/README.md +++ b/terraform/litellm/gcp/README.md @@ -321,6 +321,43 @@ launcher reads these variables, starts the pooler once per instance before uvicorn forks the workers and hands them its loopback `DATABASE_URL`. It also honours `KEEPALIVE_TIMEOUT` from `gateway_extra_env` the way the image does +### Collector sidecar + +`collector_enabled = true` adds a `spend-collector` container to the gateway +Cloud Run service that runs `python -m litellm.proxy.collector` from the gateway +image, and sets `LITELLM_COLLECTOR_ENABLED=true` on the gateway so its +uvicorn workers ship spend events (SpendLogs writes, key/team/user spend +updates, budget alerts) to the sidecar instead of running that pipeline in +the request path. This is the Terraform counterpart of helm's +`gateway.collector`. The default (`false`) leaves the service exactly as +before. It is independent of the metrics sidecars above, whose GMP scraper +already owns the `collector` container name. + +Containers in one Cloud Run instance share localhost, so the sidecar listens +on loopback TCP (`tcp://127.0.0.1:${collector_port}`, default 4010) +instead of the Unix socket helm uses; the proxy rejects any non-loopback +address. The sidecar runs the same Redis CA + `DATABASE_URL` bootstrap as +the gateway container, gets the same database, Redis, master-key, license, +proxy config, and `gateway_extra_env` / `gateway_extra_secrets` values, and +runs with `LITELLM_JOB_ROLE=collector`. When it is unreachable the +gateway falls back to in-process spend tracking. + +```hcl +collector_enabled = true +# collector_cpu = "1000m" # added on top of gateway_cpu +# collector_memory = "2Gi" # added on top of gateway_memory +# collector_buffer_size = 1000 +# collector_on_unavailable = "fallback" # or "drop" +# collector_drain_timeout_seconds = 10 +``` + +Cloud Run allocates CPU per instance while requests are in flight, and the +sidecar shares that allocation. Spend events are shipped right after each +response, so this works with request-based billing, but keep +`gateway_min_instances >= 1` if spend must keep draining while an instance +is otherwise idle. Variable names match the AWS stack; only the resource +units differ (Cloud Run strings vs Fargate units) + ## Tenant deployment Every resource the stack creates is named `${tenant}-litellm-${env}` (or diff --git a/terraform/litellm/gcp/cloudrun.tf b/terraform/litellm/gcp/cloudrun.tf index 78d4ffa2152..d0b32a367d6 100644 --- a/terraform/litellm/gcp/cloudrun.tf +++ b/terraform/litellm/gcp/cloudrun.tf @@ -177,6 +177,34 @@ locals { [local.backend_launch_cmd], )) + collector_address = "tcp://127.0.0.1:${var.collector_port}" + collector_env_kv = var.collector_enabled ? [ + { name = "LITELLM_COLLECTOR_ENABLED", value = "true" }, + { name = "LITELLM_COLLECTOR_ADDRESS", value = local.collector_address }, + { name = "LITELLM_COLLECTOR_BUFFER_SIZE", value = tostring(var.collector_buffer_size) }, + { name = "LITELLM_COLLECTOR_ON_UNAVAILABLE", value = var.collector_on_unavailable }, + { name = "LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS", value = tostring(var.collector_drain_timeout_seconds) }, + ] : [] + + gateway_env_kv = concat(local.shared_env_kv, local.gateway_otel_env_kv, local.billing_metrics_env_kv, local.gateway_extra_env_kv, local.proxy_config_env, local.metrics_env_kv, local.gateway_pool_env, local.collector_env_kv) + gateway_env_secrets = concat(local.shared_env_secrets, local.otel_env_secrets, local.billing_metrics_env_secrets, local.gateway_extra_secret_kv) + + collector_env_kv_all = concat( + local.shared_env_kv, + local.gateway_extra_env_kv, + local.proxy_config_env, + local.gateway_pool_env, + local.collector_env_kv, + [{ name = "LITELLM_JOB_ROLE", value = "collector" }], + ) + collector_env_secrets = concat(local.shared_env_secrets, local.gateway_extra_secret_kv) + + collector_args = join(" && ", concat( + local.redis_ca_fragment, + local.database_url_fragment, + ["exec python -m litellm.proxy.collector"], + )) + # Env shipped to the migrations Job. The migrations image runs run.py # which assembles DATABASE_URL from these discrete vars itself, so we # only need writer-side DB env (no read replica, no proxy_config, no @@ -203,6 +231,13 @@ resource "google_cloud_run_v2_service" "gateway" { labels = local.labels deletion_protection = false + lifecycle { + precondition { + condition = !var.collector_enabled || var.gateway_metrics_port == null || var.collector_port != var.gateway_metrics_port + error_message = "collector_port and gateway_metrics_port must differ: both sidecars bind loopback in the same instance." + } + } + template { service_account = google_service_account.runtime.email max_instance_request_concurrency = var.gateway_max_instance_request_concurrency @@ -235,7 +270,7 @@ resource "google_cloud_run_v2_service" "gateway" { } dynamic "env" { - for_each = concat(local.shared_env_kv, local.gateway_otel_env_kv, local.billing_metrics_env_kv, local.gateway_extra_env_kv, local.proxy_config_env, local.metrics_env_kv, local.gateway_pool_env) + for_each = local.gateway_env_kv content { name = env.value.name value = env.value.value @@ -243,7 +278,7 @@ resource "google_cloud_run_v2_service" "gateway" { } dynamic "env" { - for_each = concat(local.shared_env_secrets, local.otel_env_secrets, local.billing_metrics_env_secrets, local.gateway_extra_secret_kv) + for_each = local.gateway_env_secrets content { name = env.value.name value_source { @@ -357,6 +392,52 @@ resource "google_cloud_run_v2_service" "gateway" { } } + dynamic "containers" { + for_each = var.collector_enabled ? [1] : [] + content { + name = "spend-collector" + image = local.gateway_image + command = ["sh", "-c"] + args = [local.collector_args] + + resources { + limits = { + cpu = var.collector_cpu + memory = var.collector_memory + } + } + + dynamic "env" { + for_each = local.collector_env_kv_all + content { + name = env.value.name + value = env.value.value + } + } + + dynamic "env" { + for_each = local.collector_env_secrets + content { + name = env.value.name + value_source { + secret_key_ref { + secret = env.value.secret + version = env.value.version + } + } + } + } + + dynamic "volume_mounts" { + for_each = local.proxy_config_enabled ? [1] : [] + content { + name = local.proxy_config_volume + mount_path = local.proxy_config_mount_path + } + } + } + } + dynamic "volumes" { for_each = local.proxy_config_enabled ? [1] : [] content { diff --git a/terraform/litellm/gcp/tests/collector.tftest.hcl b/terraform/litellm/gcp/tests/collector.tftest.hcl new file mode 100644 index 00000000000..7a5ea781acb --- /dev/null +++ b/terraform/litellm/gcp/tests/collector.tftest.hcl @@ -0,0 +1,165 @@ +# Plan-only coverage for the opt-in collector sidecar on the gateway Cloud +# Run service. `mock_provider` keeps this offline: no GCP credentials, no API +# calls. Run from terraform/litellm/gcp with `terraform test`. + +mock_provider "google" {} +mock_provider "google-beta" {} +mock_provider "random" {} + +variables { + project_id = "acme-test" + region = "us-central1" + tenant = "acme" + env = "test" + allow_plaintext_lb = true +} + +run "disabled_by_default_leaves_the_service_untouched" { + command = plan + + assert { + condition = [for c in google_cloud_run_v2_service.gateway[0].template[0].containers : c.name] == ["gateway"] + error_message = "The gateway service must stay single-container unless collector_enabled is set." + } + + assert { + condition = !anytrue([ + for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : startswith(e.name, "LITELLM_COLLECTOR_") + ]) + error_message = "No LITELLM_COLLECTOR_* env may reach the gateway while the sidecar is disabled." + } +} + +run "enabled_adds_a_sidecar_that_shares_the_gateway_transport" { + command = plan + + variables { + collector_enabled = true + collector_port = 4321 + collector_buffer_size = 250 + collector_on_unavailable = "drop" + collector_cpu = "500m" + collector_memory = "1Gi" + gateway_extra_env = { OPENAI_API_BASE = "https://example.invalid" } + gateway_extra_secrets = { OPENAI_API_KEY = "projects/acme-test/secrets/openai-api-key" } + } + + assert { + condition = [for c in google_cloud_run_v2_service.gateway[0].template[0].containers : c.name] == ["gateway", "spend-collector"] + error_message = "Enabling the sidecar must append a spend-collector container after the gateway container." + } + + assert { + condition = alltrue([ + for c in google_cloud_run_v2_service.gateway[0].template[0].containers : ( + { for e in c.env : e.name => e.value }["LITELLM_COLLECTOR_ENABLED"] == "true" && + { for e in c.env : e.name => e.value }["LITELLM_COLLECTOR_ADDRESS"] == "tcp://127.0.0.1:4321" && + { for e in c.env : e.name => e.value }["LITELLM_COLLECTOR_BUFFER_SIZE"] == "250" && + { for e in c.env : e.name => e.value }["LITELLM_COLLECTOR_ON_UNAVAILABLE"] == "drop" && + { for e in c.env : e.name => e.value }["LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS"] == "10" + ) + ]) + error_message = "Gateway and sidecar must agree on the loopback address and the collector knobs." + } + + assert { + condition = ( + google_cloud_run_v2_service.gateway[0].template[0].containers[1].image == local.gateway_image && + google_cloud_run_v2_service.gateway[0].template[0].containers[1].command == tolist(["sh", "-c"]) && + endswith(google_cloud_run_v2_service.gateway[0].template[0].containers[1].args[0], " && exec python -m litellm.proxy.collector") && + strcontains(google_cloud_run_v2_service.gateway[0].template[0].containers[1].args[0], "export DATABASE_URL=") && + strcontains(google_cloud_run_v2_service.gateway[0].template[0].containers[1].args[0], "REDIS_SSL_CA_CERTS") && + { for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e.name => e.value }["LITELLM_JOB_ROLE"] == "collector" + ) + error_message = "The sidecar must run litellm.proxy.collector from the gateway image with the same Redis CA + DATABASE_URL bootstrap as the gateway." + } + + assert { + condition = ( + length(google_cloud_run_v2_service.gateway[0].template[0].containers[1].ports) == 0 && + google_cloud_run_v2_service.gateway[0].template[0].containers[1].resources[0].limits.cpu == "500m" && + google_cloud_run_v2_service.gateway[0].template[0].containers[1].resources[0].limits.memory == "1Gi" + ) + error_message = "The sidecar must not claim the ingress port and must carry its own resource limits." + } + + assert { + condition = ( + { for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e.name => e.value }["OPENAI_API_BASE"] == "https://example.invalid" && + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e.name], "DATABASE_HOST") && + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e.name], "REDIS_HOST") && + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e.name if length(e.value_source) > 0], "LITELLM_MASTER_KEY") && + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e.name if length(e.value_source) > 0], "DATABASE_PASSWORD") && + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e.name if length(e.value_source) > 0], "OPENAI_API_KEY") + ) + error_message = "The sidecar must receive the gateway's database, Redis, and Secret Manager env plus gateway_extra_env / gateway_extra_secrets." + } +} + +run "coexists_with_the_metrics_sidecars" { + command = plan + + variables { + collector_enabled = true + gateway_metrics_port = 4001 + } + + assert { + condition = [for c in google_cloud_run_v2_service.gateway[0].template[0].containers : c.name] == ["gateway", "metrics", "collector", "spend-collector"] + error_message = "The spend collector must keep its own container name next to the GMP metrics collector." + } + + assert { + condition = ( + { for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : e.name => e.value }["PROMETHEUS_MULTIPROC_DIR"] == local.metrics_multiproc_dir && + { for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : e.name => e.value }["LITELLM_COLLECTOR_ENABLED"] == "true" + ) + error_message = "The gateway container must keep both the metrics and the collector env when both sidecars are on." + } +} + +run "sidecars_must_not_share_a_loopback_port" { + command = plan + + variables { + collector_enabled = true + collector_port = 4001 + gateway_metrics_port = 4001 + } + + expect_failures = [ + google_cloud_run_v2_service.gateway, + ] +} + +run "collector_cannot_take_the_metrics_sidecar_health_port" { + command = plan + + variables { + collector_enabled = true + collector_port = 13133 + } + + expect_failures = [ + var.collector_port, + ] +} + +run "proxy_config_is_mounted_into_the_sidecar_too" { + command = plan + + variables { + collector_enabled = true + proxy_config = { model_list = [] } + } + + assert { + condition = alltrue([ + for c in google_cloud_run_v2_service.gateway[0].template[0].containers : ( + [for m in c.volume_mounts : m.name] == [local.proxy_config_volume] && + contains([for e in c.env : e.name], "CONFIG_FILE_PATH") + ) + ]) + error_message = "Both containers must mount the proxy-config GCS volume and point CONFIG_FILE_PATH at it." + } +} diff --git a/terraform/litellm/gcp/variables.tf b/terraform/litellm/gcp/variables.tf index f298a0431c0..412f919ab89 100644 --- a/terraform/litellm/gcp/variables.tf +++ b/terraform/litellm/gcp/variables.tf @@ -656,3 +656,73 @@ variable "billing_metrics_ca_cert_pem" { default = "" sensitive = true } + +# ---------- Collector sidecar ---------- +# +# Opt-in offload of spend tracking from the gateway's uvicorn workers to a +# `python -m litellm.proxy.collector` sidecar container in the same Cloud Run +# instance (helm's `gateway.collector`, mirrors the AWS stack). Containers +# in one instance share localhost, so the sidecar listens on loopback TCP. +# Disabled (the default) adds nothing to the service. + +variable "collector_enabled" { + description = "Run the collector sidecar next to the gateway container and have the gateway ship spend events to it (sets LITELLM_COLLECTOR_ENABLED=true on both). The sidecar shares the instance's request-based CPU allocation, so pair it with a non-zero gateway_min_instances if spend must keep flowing between requests." + type = bool + default = false +} + +variable "collector_port" { + description = "Loopback TCP port the sidecar listens on (LITELLM_COLLECTOR_ADDRESS=tcp://127.0.0.1:)." + type = number + default = 4010 + + validation { + condition = var.collector_port >= 1024 && var.collector_port <= 65535 && !contains([4000, 13133], var.collector_port) + error_message = "collector_port must be in 1024-65535 and not 4000 (the gateway port) or 13133 (the metrics sidecar health port)." + } +} + +variable "collector_cpu" { + description = "Cloud Run CPU limit for the sidecar container, on top of gateway_cpu. Matches helm's collector.resources.limits.cpu." + type = string + default = "1000m" +} + +variable "collector_memory" { + description = "Cloud Run memory limit for the sidecar container, on top of gateway_memory. Matches helm's collector.resources.limits.memory." + type = string + default = "2Gi" +} + +variable "collector_buffer_size" { + description = "Per-worker in-memory queue of spend events waiting to be shipped to the sidecar (LITELLM_COLLECTOR_BUFFER_SIZE)." + type = number + default = 1000 + + validation { + condition = var.collector_buffer_size >= 1 + error_message = "collector_buffer_size must be >= 1." + } +} + +variable "collector_on_unavailable" { + description = "What the gateway does with spend events when the sidecar is unreachable or the buffer is full (LITELLM_COLLECTOR_ON_UNAVAILABLE): `fallback` runs the pipeline in-process, `drop` discards them." + type = string + default = "fallback" + + validation { + condition = contains(["fallback", "drop"], var.collector_on_unavailable) + error_message = "collector_on_unavailable must be one of: fallback, drop." + } +} + +variable "collector_drain_timeout_seconds" { + description = "Seconds a gateway worker waits on shutdown for its buffered spend events to reach the sidecar (LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS)." + type = number + default = 10 + + validation { + condition = var.collector_drain_timeout_seconds > 0 + error_message = "collector_drain_timeout_seconds must be > 0." + } +} diff --git a/tests/test_gateway/test_launch.py b/tests/test_gateway/test_launch.py index 6039aaafc2b..a783ce6ac7e 100644 --- a/tests/test_gateway/test_launch.py +++ b/tests/test_gateway/test_launch.py @@ -3,6 +3,7 @@ import socket import sys import textwrap import urllib.parse +from collections.abc import Iterator from pathlib import Path from typing import Final, cast from unittest.mock import MagicMock, patch @@ -66,7 +67,7 @@ def _query(url: str) -> dict[str, str]: @pytest.fixture -def password_env(monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: +def password_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[dict[str, str]]: for var in ( "DATABASE_URL", "IAM_TOKEN_DB_AUTH", @@ -78,7 +79,8 @@ def password_env(monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: monkeypatch.delenv(var) for var, value in DB_ENV.items(): monkeypatch.setenv(var, value) - return dict(DB_ENV) + yield dict(DB_ENV) + os.environ.pop("DATABASE_URL", None) def _minted_iam_token(token: str): diff --git a/tests/test_litellm/caching/test_caching_handler.py b/tests/test_litellm/caching/test_caching_handler.py index 8e0bc200012..071b99850f6 100644 --- a/tests/test_litellm/caching/test_caching_handler.py +++ b/tests/test_litellm/caching/test_caching_handler.py @@ -658,3 +658,38 @@ def test_async_cache_write_completes_when_asyncio_run_closes_the_loop(monkeypatc asyncio.run(_short_lived_script()) assert len(writes) == 1 + + +@pytest.mark.asyncio +async def test_cache_hit_records_the_looked_up_key_as_the_preset_cache_key(monkeypatch): + """The spend log for a cache hit must reuse the key the lookup already computed instead of hashing again.""" + import litellm + from litellm.caching.caching import Cache + from litellm.types.utils import CallTypes + + async def acompletion(**kwargs): + return None + + monkeypatch.setattr(litellm, "cache", Cache(type="local")) + kwargs = {"model": "gpt-5.4", "messages": [{"role": "user", "content": "hello"}], "caching": True} + await litellm.cache.async_add_cache( + litellm.ModelResponse(choices=[{"message": {"role": "assistant", "content": "hi"}}]), **kwargs + ) + handler = LLMCachingHandler(original_function=acompletion, request_kwargs=kwargs, start_time=datetime.now()) + logging_obj = _build_logging_obj(CallTypes.acompletion.value, stream=False) + logging_obj.async_success_handler = AsyncMock() + + hit = await handler._async_get_cache( + model="gpt-5.4", + original_function=acompletion, + logging_obj=logging_obj, + start_time=datetime.now(), + call_type=CallTypes.acompletion.value, + kwargs=kwargs, + args=(), + ) + + assert hit is not None and hit.cached_result is not None + assert handler.preset_cache_key is not None + assert logging_obj.litellm_params["preset_cache_key"] == handler.preset_cache_key + assert hit.cached_result._hidden_params["cache_key"] == handler.preset_cache_key diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index eff892f2d80..c96cfc3ee4a 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -5,14 +5,20 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth +from litellm.proxy.collector import SpendEventConsumer +from litellm.proxy.db.spend_log_tool_index import response_tool_call_names from litellm.proxy.hooks.proxy_track_cost_callback import ( _get_budget_reservation_from_metadata, _ProxyDBLogger, _should_track_cost_callback, _update_database_and_spend_counters, + run_spend_event, ) -from litellm.types.utils import CallTypes, Usage +from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event +from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress +from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload +from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, Usage @pytest.mark.asyncio @@ -2096,3 +2102,248 @@ async def test_spend_counters_keep_every_granted_group_when_the_deployment_is_un ) assert charged == ("premium", "tier0") + + +def _offload_kwargs() -> dict: + big_prompt = "x" * 10_000 + reservation = {"reserved_cost": 0.5, "entries": [{"counter_key": "key:hash-1", "reserved_cost": 0.5}]} + return { + "litellm_call_id": "call-1", + "call_type": "acompletion", + "model": "gpt-4o", + "custom_llm_provider": "openai", + "stream": False, + "cache_hit": None, + "response_cost": 0.0125, + "completion_start_time": datetime(2026, 1, 1, 0, 0, 1), + "messages": [{"role": "user", "content": big_prompt}], + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + "litellm_params": { + "api_base": "https://api.openai.com", + "preset_cache_key": None, + "proxy_server_request": {"body": {"messages": [{"role": "user", "content": big_prompt}]}}, + "metadata": { + "user_api_key": "hash-1", + "user_api_key_hash": "hash-1", + "user_api_key_alias": "alias-1", + "user_api_key_user_id": "user-1", + "user_api_key_team_id": "team-1", + "user_api_key_org_id": "org-1", + "user_api_key_end_user_id": "end-user-1", + "user_api_key_auth": UserAPIKeyAuth(api_key="hash-1", budget_reservation=reservation), + "model_group": "gpt-4o", + "model_info": {"id": "deployment-1"}, + "tags": ["tag-a"], + }, + }, + "standard_logging_object": { + "id": "chatcmpl-1", + "trace_id": "trace-1", + "response_cost": 0.0125, + "model": "gpt-4o-2024-08-06", + "model_id": "deployment-1", + "model_group": "gpt-4o", + "api_base": "https://api.openai.com", + "custom_llm_provider": "openai", + "prompt_tokens": 5000, + "completion_tokens": 4000, + "total_tokens": 9000, + "request_tags": ["tag-a"], + "request_model_access_groups": ["premium"], + "messages": [{"role": "user", "content": big_prompt}], + "response": {"choices": [{"message": {"content": "y" * 10_000}}]}, + "model_parameters": {"temperature": 0.1}, + "metadata": { + "user_api_key_hash": "hash-1", + "user_api_key_end_user_id": "end-user-1", + "usage_object": {"prompt_tokens": 5000, "completion_tokens": 4000, "total_tokens": 9000}, + }, + "hidden_params": {"litellm_overhead_time_ms": 3}, + "model_map_information": {}, + "cost_breakdown": {"input_cost": 0.0125, "output_cost": 0.0}, + }, + } + + +def _offload_response() -> ModelResponse: + return ModelResponse( + id="chatcmpl-1", + model="gpt-4o-2024-08-06", + choices=[ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call-1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + ], + }, + "finish_reason": "tool_calls", + } + ], + usage=Usage(prompt_tokens=5000, completion_tokens=4000, total_tokens=9000), + ) + + +class _RecordingHandler: + def __init__(self) -> None: + self.lines: list[bytes] = [] # mutable-ok: test double records the events the sidecar received + + async def __call__(self, line: bytes) -> None: + self.lines.append(line) + + +async def _no_fallback(line: bytes) -> None: + raise AssertionError("the sidecar was reachable, nothing should fall back") + + +@pytest.mark.asyncio +async def test_async_log_success_event_hands_the_sidecar_a_compact_event_and_skips_the_pipeline(tmp_path): + handler = _RecordingHandler() + consumer = SpendEventConsumer(handler) + address = UnixAddress(path=str(tmp_path / "spend.sock")) + server = await consumer.serve(address) + producer = SpendEventProducer( + address=address, on_unavailable="fallback", buffer_size=10, connect_timeout=1.0, fallback=_no_fallback + ) + logger = _ProxyDBLogger(producer) + + with ( + patch( # test-quality-ok: the callback imports this from proxy_server inside its body, so there is no injection seam + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as mock_proxy_logging, + patch( # test-quality-ok: same function-body import, no injection seam + "litellm.proxy.proxy_server.increment_spend_counters", new_callable=AsyncMock + ) as counters, + ): + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock() + await logger.async_log_success_event(_offload_kwargs(), _offload_response(), datetime.now(), datetime.now()) + await producer.close(drain_timeout=5.0) + server.close() + assert await consumer.drain(timeout=5.0) == 0 + + mock_proxy_logging.db_spend_update_writer.update_database.assert_not_awaited() + counters.assert_not_awaited() + assert producer.stats().sent == 1 + assert len(handler.lines) == 1 + assert len(handler.lines[0]) < 4_000 + event = decode_spend_event(handler.lines[0]) + assert not isinstance(event, SpendEventDecodeError) + assert event.litellm_params["metadata"]["user_api_key_team_id"] == "team-1" + assert event.response_cost == 0.0125 + + +@pytest.mark.asyncio +async def test_async_log_success_event_keeps_batch_retrieves_in_process(): + producer = SpendEventProducer( + address=UnixAddress(path="/nonexistent/spend.sock"), + on_unavailable="drop", + buffer_size=10, + connect_timeout=1.0, + fallback=_no_fallback, + ) + logger = _ProxyDBLogger(producer) + kwargs = {**_offload_kwargs(), "call_type": CallTypes.aretrieve_batch.value} + completed_batch = LiteLLMBatch( + id="batch_abc", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="file-in", + output_file_id="file-out", + object="batch", + status="completed", + ) + + with ( + patch( # test-quality-ok: the callback imports this from proxy_server inside its body, so there is no injection seam + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as mock_proxy_logging, + patch( # test-quality-ok: same function-body import, no injection seam + "litellm.proxy.proxy_server.increment_spend_counters", new_callable=AsyncMock + ), + patch( # test-quality-ok: same function-body import, no injection seam + "litellm.proxy.proxy_server.update_cache", new_callable=AsyncMock + ), + ): + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock(return_value=True) + mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock() + await logger.async_log_success_event(kwargs, completed_batch, datetime.now(), datetime.now()) + + mock_proxy_logging.db_spend_update_writer.update_database.assert_awaited_once() + assert producer.stats().queued == 0 + + +async def _spend_row_written_by(run) -> tuple[SpendLogsPayload, dict, tuple[str, ...]]: + with ( + patch( # test-quality-ok: the callback imports this from proxy_server inside its body, so there is no injection seam + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as mock_proxy_logging, + patch( # test-quality-ok: same function-body import, no injection seam + "litellm.proxy.proxy_server.increment_spend_counters", new_callable=AsyncMock + ) as counters, + patch( # test-quality-ok: same function-body import, no injection seam + "litellm.proxy.proxy_server.update_cache", new_callable=AsyncMock + ), + ): + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock(return_value=True) + mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock() + await run() + mock_proxy_logging.db_spend_update_writer.update_database.assert_awaited_once() + written = mock_proxy_logging.db_spend_update_writer.update_database.await_args.kwargs + counters.assert_awaited_once() + counted = dict(counters.await_args.kwargs) + + row = get_logging_payload( + kwargs=written["kwargs"], + response_obj=written["completion_response"], + start_time=written["start_time"], + end_time=written["end_time"], + ) + return row, counted, response_tool_call_names(written["completion_response"]) + + +@pytest.mark.asyncio +async def test_sidecar_writes_the_same_spend_row_and_counters_as_the_in_process_path(): + start_time = datetime(2026, 1, 1, 0, 0, 0) + end_time = datetime(2026, 1, 1, 0, 0, 2) + + async def in_process() -> None: + await _ProxyDBLogger().async_log_success_event(_offload_kwargs(), _offload_response(), start_time, end_time) + + async def via_sidecar() -> None: + line = build_spend_event(_offload_kwargs(), _offload_response(), start_time, end_time, store_bodies=False) + assert isinstance(line, bytes) + await run_spend_event(line) + + in_process_row, in_process_counters, in_process_tools = await _spend_row_written_by(in_process) + sidecar_row, sidecar_counters, sidecar_tools = await _spend_row_written_by(via_sidecar) + + assert sidecar_row == in_process_row + assert in_process_row["spend"] == 0.0125 + assert in_process_row["team_id"] == "team-1" + assert in_process_row["end_user"] == "end-user-1" + assert in_process_row["total_tokens"] == 9000 + assert in_process_row["model_id"] == "deployment-1" + assert in_process_row["request_tags"] == '["tag-a"]' + assert in_process_row["messages"] == "{}" + assert in_process_row["response"] == "{}" + assert sidecar_counters == in_process_counters + assert in_process_counters["token"] == "hash-1" + assert in_process_counters["response_cost"] == 0.0125 + assert in_process_counters["budget_reservation"]["reserved_cost"] == 0.5 + assert in_process_counters["model_access_groups"] == ("premium",) + assert sidecar_tools == in_process_tools == ("get_weather",) + + +@pytest.mark.asyncio +async def test_sidecar_ignores_an_undecodable_event(): # test-quality-ok: a discarded event has no observable output other than the DB writer never being reached + with ( + patch( # test-quality-ok: the callback imports this from proxy_server inside its body, so there is no injection seam + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as mock_proxy_logging + ): + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock() + await run_spend_event(b"garbage\n") + mock_proxy_logging.db_spend_update_writer.update_database.assert_not_awaited() diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index dafe686c4f2..deb7289d2d1 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -253,6 +253,38 @@ async def test_flush_spend_logs_queue_on_shutdown_swallows_drain_errors(monkeypa await ps._flush_spend_logs_queue_on_shutdown() +@pytest.mark.asyncio +async def test_flush_spend_counters_on_shutdown_commits_buffered_spend(monkeypatch): + fake_prisma = MagicMock() + monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) + commit = AsyncMock() + monkeypatch.setattr(ps.proxy_logging_obj.db_spend_update_writer, "db_update_spend_transaction_handler", commit) + + await ps.flush_spend_counters_on_shutdown() + + observed = { + "commit_calls": commit.await_count, + "commit_prisma": commit.await_args.kwargs["prisma_client"] is fake_prisma, + "commit_proxy_logging": commit.await_args.kwargs["proxy_logging_obj"] is ps.proxy_logging_obj, + } + assert observed == {"commit_calls": 1, "commit_prisma": True, "commit_proxy_logging": True} + + +@pytest.mark.asyncio +async def test_flush_spend_counters_on_shutdown_logs_and_swallows_commit_errors(monkeypatch, caplog): + monkeypatch.setattr(ps, "prisma_client", MagicMock(), raising=False) + monkeypatch.setattr( + ps.proxy_logging_obj.db_spend_update_writer, + "db_update_spend_transaction_handler", + AsyncMock(side_effect=RuntimeError("db gone")), + ) + + with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): + await ps.flush_spend_counters_on_shutdown() + + assert "Error flushing spend counters on shutdown: db gone" in caplog.text + + # --------------------------------------------------------------------------- # _initialize_shared_aiohttp_session # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_event.py b/tests/test_litellm/proxy/spend_tracking/test_spend_event.py new file mode 100644 index 00000000000..ff449235582 --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_event.py @@ -0,0 +1,213 @@ +import json +from datetime import datetime +from typing import Final + +import pytest + +import litellm +from litellm.caching.caching import Cache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.spend_tracking.spend_event import ( + CACHE_OFF_KEY, + SpendEventBuildError, + SpendEventDecodeError, + build_spend_event, + decode_spend_event, + is_offloadable_success, + spend_event_callback_args, +) +from litellm.types.utils import LiteLLMBatch, ModelResponse, Usage + +_BIG_PROMPT: Final = "x" * 20_000 +_RESERVATION: Final = { + "reserved_cost": 0.5, + "entries": [{"counter_key": "key:hash", "reserved_cost": 0.5}], + "finalized": False, + "input_cost": 0.1, + "input_tokens": 5000, +} + + +def _response(tool_name: str | None = None) -> ModelResponse: + tool_calls: Final = ( + [{"id": "call-1", "type": "function", "function": {"name": tool_name, "arguments": "{}"}}] + if tool_name is not None + else None + ) + return ModelResponse( + id="chatcmpl-1", + model="gpt-4o-2024-08-06", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "y" * 20_000, "tool_calls": tool_calls}, + "finish_reason": "tool_calls" if tool_name else "stop", + } + ], + usage=Usage(prompt_tokens=5000, completion_tokens=4000, total_tokens=9000), + ) + + +def _success_kwargs(preset_cache_key: str | None = "preset-key") -> dict: + return { + "litellm_call_id": "call-1", + "call_type": "acompletion", + "model": "gpt-4o", + "custom_llm_provider": "openai", + "stream": False, + "cache_hit": None, + "response_cost": 0.0125, + "completion_start_time": datetime(2026, 1, 1, 0, 0, 1), + "messages": [{"role": "user", "content": _BIG_PROMPT}], + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + "litellm_params": { + "api_base": "https://api.openai.com", + "preset_cache_key": preset_cache_key, + "proxy_server_request": {"body": {"messages": [{"role": "user", "content": _BIG_PROMPT}]}}, + "metadata": { + "user_api_key": "hash-1", + "user_api_key_user_id": "user-1", + "user_api_key_team_id": "team-1", + "user_api_key_org_id": "org-1", + "user_api_key_end_user_id": "end-user-1", + "user_api_key_auth": UserAPIKeyAuth(api_key="hash-1", budget_reservation=dict(_RESERVATION)), + "model_group": "gpt-4o", + "model_info": {"id": "deployment-1"}, + "tags": ["tag-a"], + "litellm_parent_otel_span": object(), + }, + }, + "standard_logging_object": { + "response_cost": 0.0125, + "model": "gpt-4o-2024-08-06", + "model_id": "deployment-1", + "request_tags": ["tag-a"], + "request_model_access_groups": ["premium"], + "messages": [{"role": "user", "content": _BIG_PROMPT}], + "response": {"choices": [{"message": {"content": "y" * 20_000}}]}, + "model_parameters": {"temperature": 0.1}, + "metadata": {"user_api_key_hash": "hash-1", "usage_object": {"prompt_tokens": 5000}}, + "hidden_params": {"litellm_overhead_time_ms": 3}, + "model_map_information": {}, + }, + } + + +def _build(kwargs: dict, response: object, store_bodies: bool = False) -> bytes: + line: Final = build_spend_event( + kwargs, response, datetime(2026, 1, 1), datetime(2026, 1, 1, 0, 0, 2), store_bodies=store_bodies + ) + assert isinstance(line, bytes) + return line + + +def test_event_is_compact_and_omits_bodies_by_default(): + line: Final = _build(_success_kwargs(), _response(tool_name="get_weather")) + assert line.endswith(b"\n") + assert len(line) < 4_000 + assert _BIG_PROMPT.encode() not in line + assert b"yyyy" not in line + decoded: Final = json.loads(line) + assert "messages" not in decoded["standard_logging_object"] + assert "response" not in decoded["standard_logging_object"] + assert decoded["litellm_params"]["proxy_server_request"] is None + + +def test_event_carries_bodies_when_spend_logs_store_them(): + line: Final = _build(_success_kwargs(), _response(), store_bodies=True) + decoded: Final = json.loads(line) + assert decoded["standard_logging_object"]["messages"][0]["content"] == _BIG_PROMPT + assert decoded["standard_logging_object"]["response"]["choices"][0]["message"]["content"] == "y" * 20_000 + assert decoded["litellm_params"]["proxy_server_request"]["body"]["messages"][0]["content"] == _BIG_PROMPT + + +def test_round_trip_preserves_identity_usage_reservation_and_tools(): + line: Final = _build(_success_kwargs(), _response(tool_name="get_weather")) + event: Final = decode_spend_event(line) + assert not isinstance(event, SpendEventDecodeError) + args: Final = spend_event_callback_args(event) + + metadata: Final = args.kwargs["litellm_params"]["metadata"] + assert metadata is not None + assert (metadata["user_api_key"], metadata["user_api_key_team_id"], metadata["user_api_key_org_id"]) == ( + "hash-1", + "team-1", + "org-1", + ) + assert metadata["user_api_key_budget_reservation"] == _RESERVATION + assert "user_api_key_auth" not in metadata + assert "litellm_parent_otel_span" not in metadata + assert args.kwargs["standard_logging_object"]["request_model_access_groups"] == ["premium"] + assert args.kwargs["standard_logging_object"]["response_cost"] == 0.0125 + assert args.kwargs["tools"] == ({"type": "function", "function": {"name": "get_weather"}},) + assert args.kwargs["completion_start_time"] == datetime(2026, 1, 1, 0, 0, 1) + assert (args.start_time, args.end_time) == (datetime(2026, 1, 1), datetime(2026, 1, 1, 0, 0, 2)) + assert args.response_obj is not None + assert args.response_obj["id"] == "chatcmpl-1" + assert args.response_obj["usage"]["prompt_tokens"] == 5000 + assert args.response_obj["usage"]["completion_tokens"] == 4000 + tool_calls: Final = args.response_obj["choices"][0]["message"]["tool_calls"] + assert [call["function"]["name"] for call in tool_calls] == ["get_weather"] + assert "complete_streaming_response" not in args.kwargs + + +def test_streaming_event_reconstructs_complete_streaming_response(): + kwargs: Final = {**_success_kwargs(), "stream": True, "complete_streaming_response": _response()} + event: Final = decode_spend_event(_build(kwargs, _response())) + assert not isinstance(event, SpendEventDecodeError) + args: Final = spend_event_callback_args(event) + assert args.kwargs["stream"] is True + assert args.kwargs["complete_streaming_response"] == args.response_obj + + +class _HashingCache(Cache): + def __init__(self) -> None: + pass + + def get_cache_key(self, **kwargs) -> str: + raise AssertionError("the fast path must not hash the request body") + + +@pytest.mark.parametrize( + ("cache", "preset", "expected"), + [ + (None, "preset-key", CACHE_OFF_KEY), + (_HashingCache(), "preset-key", "preset-key"), + (_HashingCache(), None, None), + ], +) +def test_event_reuses_preset_cache_key_and_never_hashes(monkeypatch, cache, preset, expected): + monkeypatch.setattr(litellm, "cache", cache) + decoded: Final = json.loads(_build(_success_kwargs(preset_cache_key=preset), _response())) + assert decoded["litellm_params"]["preset_cache_key"] == expected + + +def test_unbuildable_kwargs_fall_back_to_in_process_tracking(): + kwargs: Final = {**_success_kwargs(), "response_cost": "not-a-number"} + assert isinstance( + build_spend_event(kwargs, _response(), datetime.now(), datetime.now(), False), SpendEventBuildError + ) + + +def test_undecodable_line_is_an_error_value(): + assert isinstance(decode_spend_event(b'{"version": 2}\n'), SpendEventDecodeError) + assert isinstance(decode_spend_event(b"not json\n"), SpendEventDecodeError) + + +def test_batch_retrieves_stay_in_process(): + assert is_offloadable_success(_response()) is True + assert is_offloadable_success(None) is True + assert ( + is_offloadable_success( + LiteLLMBatch( + id="batch-1", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="f", + object="batch", + status="completed", + ) + ) + is False + ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_event_producer.py b/tests/test_litellm/proxy/spend_tracking/test_spend_event_producer.py new file mode 100644 index 00000000000..adfb1251d1c --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_event_producer.py @@ -0,0 +1,359 @@ +import asyncio +from collections.abc import Awaitable, Callable +from pathlib import Path +from typing import Final + +import pytest +import uvloop + +from litellm.proxy.spend_tracking.spend_event_producer import ( + AddressError, + CollectorAddress, + CollectorSettings, + SpendEventProducer, + TcpAddress, + UnixAddress, + build_spend_event_producer, + open_collector_connection, + parse_collector_address, +) + + +class _Sidecar: + """A unix-socket server that records every line it receives, standing in for the collector.""" + + def __init__(self, path: Path, reads: bool = True, limit: int = 2**16) -> None: + self.path = path + self.reads = reads + self.limit = limit + self.lines: list[bytes] = [] # mutable-ok: test double records what the producer sent + self._server: asyncio.Server | None = None + self._stopped = asyncio.Event() + self._connections: list[asyncio.StreamWriter] = [] # mutable-ok: test double tracks peers to hang up on + + async def __aenter__(self) -> "_Sidecar": + self._server = await asyncio.start_unix_server(self._on_connection, path=str(self.path), limit=self.limit) + return self + + async def __aexit__(self, *exc: object) -> None: + self._stopped.set() + await self.hang_up() + + async def hang_up(self) -> None: + """Exit the way a stopped sidecar does: stop listening and close every producer connection.""" + assert self._server is not None + self._server.close() + for connection in self._connections: + connection.close() + await connection.wait_closed() + await self._server.wait_closed() + + async def _on_connection(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + self._connections.append(writer) + if not self.reads: + await self._stopped.wait() + return + while line := await reader.readline(): + self.lines.append(line) + writer.close() + + +class _CrashingSidecar(_Sidecar): + """Bills a few lines, then dies mid-stream with the producer's backlog still queued behind them.""" + + def __init__(self, path: Path, lines_before_crash: int) -> None: + super().__init__(path, limit=2**20) + self._lines_before_crash = lines_before_crash + + async def _on_connection(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + self._connections.append(writer) + for _ in range(self._lines_before_crash): + self.lines.append(await reader.readline()) + writer.transport.abort() + + +class _Fallback: + def __init__(self) -> None: + self.lines: list[bytes] = [] # mutable-ok: test double records what fell back to in-process + + async def __call__(self, line: bytes) -> None: + self.lines.append(line) + + +class _GatedFallback(_Fallback): + """A fallback that blocks, like a slow database write, until the test releases it.""" + + def __init__(self) -> None: + super().__init__() + self.started = asyncio.Event() + self.release = asyncio.Event() + + async def __call__(self, line: bytes) -> None: + self.started.set() + await self.release.wait() + await super().__call__(line) + + +class _StalledDrainWriter(asyncio.StreamWriter): + """Hands bytes to the real transport but never wakes ``drain()``: the loop iteration between a flush + completing and the writer task resuming, frozen in place.""" + + def __init__(self, real: asyncio.StreamWriter, reader: asyncio.StreamReader) -> None: + super().__init__(real.transport, real.transport.get_protocol(), reader, asyncio.get_running_loop()) + self._real_writer_whose_finalizer_would_close_the_transport = real + + async def drain(self) -> None: + await asyncio.Event().wait() + + +async def _open_with_stalled_drain( + address: CollectorAddress, timeout: float +) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]: + reader, writer = await open_collector_connection(address, timeout) + return reader, _StalledDrainWriter(writer, reader) + + +def _producer( + path: Path, + fallback: _Fallback, + on_unavailable="fallback", + buffer_size: int = 100, + open_connection: Callable[ + [CollectorAddress, float], Awaitable[tuple[asyncio.StreamReader, asyncio.StreamWriter]] + ] = open_collector_connection, +) -> SpendEventProducer: + return SpendEventProducer( + address=UnixAddress(path=str(path)), + on_unavailable=on_unavailable, + buffer_size=buffer_size, + connect_timeout=1.0, + fallback=fallback, + open_connection=open_connection, + ) + + +def test_parse_collector_address(): + assert parse_collector_address("unix:///var/run/litellm/collector.sock") == UnixAddress( + path="/var/run/litellm/collector.sock" + ) + assert parse_collector_address("tcp://127.0.0.1:4100") == TcpAddress(host="127.0.0.1", port=4100) + assert parse_collector_address("tcp://localhost:4100") == TcpAddress(host="localhost", port=4100) + assert parse_collector_address("tcp://[::1]:4100") == TcpAddress(host="::1", port=4100) + assert isinstance(parse_collector_address("redis://localhost:6379"), AddressError) + assert isinstance(parse_collector_address("tcp://127.0.0.1"), AddressError) + + +@pytest.mark.parametrize("address", ["tcp://0.0.0.0:4100", "tcp://10.0.0.5:4100", "tcp://collector.svc:4100"]) +def test_tcp_address_outside_loopback_is_refused(address: str): + """The socket has no authentication, so anything reachable from outside the pod would accept forged spend.""" + error: Final = parse_collector_address(address) + assert isinstance(error, AddressError) + assert "loopback" in error.reason + assert build_spend_event_producer(CollectorSettings(enabled=True, address=address), _Fallback()) is None + + +def test_gateway_produces_only_when_enabled_and_not_the_sidecar_itself(): + fallback: Final = _Fallback() + assert build_spend_event_producer(CollectorSettings(enabled=False), fallback) is None + assert build_spend_event_producer(CollectorSettings(enabled=True, job_role="collector"), fallback) is None + assert build_spend_event_producer(CollectorSettings(enabled=True, address="redis://x"), fallback) is None + assert isinstance(build_spend_event_producer(CollectorSettings(enabled=True), fallback), SpendEventProducer) + + +def test_settings_read_the_documented_env(monkeypatch): + monkeypatch.setenv("LITELLM_COLLECTOR_ENABLED", "true") + monkeypatch.setenv("LITELLM_COLLECTOR_ADDRESS", "tcp://127.0.0.1:4100") + monkeypatch.setenv("LITELLM_COLLECTOR_BUFFER_SIZE", "50") + monkeypatch.setenv("LITELLM_COLLECTOR_ON_UNAVAILABLE", "drop") + monkeypatch.setenv("LITELLM_JOB_ROLE", "collector") + settings: Final = CollectorSettings() + assert (settings.enabled, settings.address, settings.buffer_size, settings.on_unavailable) == ( + True, + "tcp://127.0.0.1:4100", + 50, + "drop", + ) + assert settings.produces is False + + +@pytest.mark.asyncio +async def test_events_reach_the_sidecar_once_and_in_order(tmp_path: Path): + fallback: Final = _Fallback() + async with _Sidecar(tmp_path / "spend.sock") as sidecar: + producer: Final = _producer(sidecar.path, fallback) + outcomes: Final = [await producer.publish(f"event-{i}\n".encode()) for i in range(20)] + await producer.close(drain_timeout=5.0) + await asyncio.sleep(0.05) + + assert outcomes == ["queued"] * 20 + assert sidecar.lines == [f"event-{i}\n".encode() for i in range(20)] + assert fallback.lines == [] + stats: Final = producer.stats() + assert (stats.queued, stats.sent, stats.fallback, stats.dropped) == (20, 20, 0, 0) + + +@pytest.mark.asyncio +async def test_unreachable_sidecar_falls_back_in_process_and_backs_off(tmp_path: Path): + fallback: Final = _Fallback() + producer: Final = _producer(tmp_path / "missing.sock", fallback) + first: Final = await producer.publish(b"event-1\n") + await asyncio.sleep(0.05) + second: Final = await producer.publish(b"event-2\n") + await producer.close(drain_timeout=5.0) + + assert first == "queued" + assert second == "fallback" + assert fallback.lines == [b"event-1\n", b"event-2\n"] + stats: Final = producer.stats() + assert (stats.sent, stats.fallback, stats.dropped, stats.connected) == (0, 2, 0, False) + + +@pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"]) +def test_sidecar_hang_up_falls_back_instead_of_losing_events( + tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] +): + async def scenario() -> tuple[list[bytes], list[bytes], tuple[int, int, int]]: + fallback: Final = _Fallback() + sidecar: Final = _Sidecar(tmp_path / "spend.sock") + async with sidecar: + producer: Final = _producer(sidecar.path, fallback) + await producer.publish(b"event-1\n") + await asyncio.sleep(0.05) + await sidecar.hang_up() + await asyncio.sleep(0.05) + await producer.publish(b"event-2\n") + await producer.close(drain_timeout=5.0) + stats: Final = producer.stats() + return sidecar.lines, fallback.lines, (stats.sent, stats.fallback, stats.dropped) + + with asyncio.Runner(loop_factory=loop_factory) as runner: + sidecar_lines, fallback_lines, counts = runner.run(scenario()) + + assert sidecar_lines == [b"event-1\n"] + assert fallback_lines == [b"event-2\n"] + assert counts == (1, 1, 0) + + +@pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"]) +def test_mid_stream_crash_never_bills_an_event_on_both_sides( + tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] +): + """Events large enough to straddle the kernel buffer, a sidecar that reads some and then drops the socket: a + failed write may only fall back when the sidecar cannot have read the whole line.""" + events: Final = tuple(f"event-{i:03d}-".encode() + b"x" * 65536 + b"\n" for i in range(64)) + + async def scenario() -> tuple[list[bytes], list[bytes], tuple[int, int, int]]: + fallback: Final = _Fallback() + sidecar: Final = _CrashingSidecar(tmp_path / "spend.sock", lines_before_crash=3) + async with sidecar: + producer: Final = _producer(sidecar.path, fallback) + for event in events: + assert await producer.publish(event) == "queued" + await asyncio.sleep(0.2) + await producer.close(drain_timeout=5.0) + stats: Final = producer.stats() + return sidecar.lines, fallback.lines, (stats.sent, stats.fallback, stats.dropped) + + with asyncio.Runner(loop_factory=loop_factory) as runner: + sidecar_lines, fallback_lines, counts = runner.run(scenario()) + + assert sidecar_lines == list(events[:3]) + assert set(sidecar_lines).isdisjoint(fallback_lines) + assert len(fallback_lines) == len(set(fallback_lines)) + assert fallback_lines[-1] == events[-1] + assert counts[0] + counts[1] == len(events) and counts[2] == 0 + assert counts[0] >= len(sidecar_lines) + + +@pytest.mark.asyncio +async def test_drain_timeout_hands_the_in_flight_event_to_fallback(tmp_path: Path): + """A sidecar that stops reading leaves one event half-written; cancelling the writer must not lose it.""" + fallback: Final = _Fallback() + stuck: Final = b"x" * (4 * 1024 * 1024) + b"\n" + async with _Sidecar(tmp_path / "spend.sock", reads=False) as sidecar: + producer: Final = _producer(sidecar.path, fallback) + assert await producer.publish(stuck) == "queued" + await asyncio.sleep(0.1) + await producer.close(drain_timeout=0.2) + + assert fallback.lines == [stuck] + stats: Final = producer.stats() + assert (stats.sent, stats.fallback, stats.connected) == (0, 1, False) + + +@pytest.mark.asyncio +async def test_shutdown_lets_the_writer_finish_a_fallback_already_in_progress(tmp_path: Path): + """Cancelling the writer while it runs the pipeline in-process must neither lose nor repeat that event.""" + fallback: Final = _GatedFallback() + producer: Final = _producer(tmp_path / "missing.sock", fallback) + assert await producer.publish(b"event-1\n") == "queued" + await asyncio.wait_for(fallback.started.wait(), 5.0) + closing: Final = asyncio.ensure_future(producer.close(drain_timeout=0.05)) + await asyncio.sleep(0.2) + assert fallback.lines == [] + fallback.release.set() + await asyncio.wait_for(closing, 5.0) + + assert fallback.lines == [b"event-1\n"] + assert producer.stats().fallback == 1 + + +@pytest.mark.asyncio +async def test_shutdown_does_not_replay_an_event_the_kernel_already_took(tmp_path: Path): + """Cancelling a drain whose bytes already left the process must not run the event a second time in-process.""" + fallback: Final = _Fallback() + async with _Sidecar(tmp_path / "spend.sock") as sidecar: + producer: Final = _producer(sidecar.path, fallback, open_connection=_open_with_stalled_drain) + assert await producer.publish(b"event-1\n") == "queued" + await asyncio.sleep(0.1) + await producer.close(drain_timeout=0.2) + await asyncio.sleep(0.05) + + assert sidecar.lines == [b"event-1\n"] + assert fallback.lines == [] + stats: Final = producer.stats() + assert (stats.fallback, stats.dropped, stats.connected) == (0, 0, False) + + +@pytest.mark.asyncio +async def test_drop_policy_counts_instead_of_running_in_process(tmp_path: Path): + fallback: Final = _Fallback() + producer: Final = _producer(tmp_path / "missing.sock", fallback, on_unavailable="drop") + await producer.publish(b"event-1\n") + await producer.close(drain_timeout=5.0) + assert await producer.publish(b"event-2\n") == "dropped" + + assert fallback.lines == [] + assert producer.stats().dropped == 2 + + +@pytest.mark.asyncio +async def test_full_buffer_applies_the_unavailable_policy_immediately(tmp_path: Path): + fallback: Final = _Fallback() + async with _Sidecar(tmp_path / "spend.sock") as sidecar: + producer: Final = _producer(sidecar.path, fallback, buffer_size=2) + outcomes: Final = [await producer.publish(f"event-{i}\n".encode()) for i in range(3)] + await producer.close(drain_timeout=5.0) + await asyncio.sleep(0.05) + + assert outcomes == ["queued", "queued", "fallback"] + assert fallback.lines == [b"event-2\n"] + assert sidecar.lines == [b"event-0\n", b"event-1\n"] + + +@pytest.mark.asyncio +async def test_close_flushes_buffered_events_then_refuses_new_ones(tmp_path: Path): + fallback: Final = _Fallback() + async with _Sidecar(tmp_path / "spend.sock") as sidecar: + producer: Final = _producer(sidecar.path, fallback) + for i in range(50): + await producer.publish(f"event-{i}\n".encode()) + assert sidecar.lines == [] + await producer.close(drain_timeout=5.0) + await asyncio.sleep(0.05) + after_close: Final = await producer.publish(b"late\n") + + assert len(sidecar.lines) == 50 + assert after_close == "fallback" + assert fallback.lines == [b"late\n"] + assert producer.stats().connected is False diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index d84d63ce1a4..df113197ec6 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -38,9 +38,9 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( _sanitize_error_information_for_spend_logs, _sanitize_guardrail_information_for_spend_logs, _sanitize_request_body_for_spend_logs_payload, - _should_store_prompts_and_responses_in_spend_logs, get_logging_payload, get_spend_logs_id, + should_store_prompts_and_responses_in_spend_logs, ) from litellm.proxy.utils import hash_token from litellm.types.utils import ( @@ -107,6 +107,48 @@ def test_get_logging_payload_maps_openai_cached_tokens_to_cache_read_input_token assert additional_usage_values["prompt_tokens_details"]["cached_tokens"] == 123 +class _HashingCache(litellm.Cache): + def __init__(self) -> None: + pass + + def get_cache_key(self, **kwargs) -> str: + raise AssertionError("a preset cache key must be reused instead of hashing the request") + + +def _cache_key_in_spend_log(monkeypatch: pytest.MonkeyPatch, cache: litellm.Cache | None, preset: str | None) -> str: + monkeypatch.setattr(litellm, "cache", cache) + payload: Final = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "x" * 10_000}], + "litellm_params": {"metadata": {"user_api_key": "test-key"}, "preset_cache_key": preset}, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + return payload["cache_key"] + + +def test_get_logging_payload_reuses_the_preset_cache_key_instead_of_hashing_the_body(monkeypatch): + assert _cache_key_in_spend_log(monkeypatch, _HashingCache(), "preset-key") == "preset-key" + + +def test_get_logging_payload_records_cache_off_without_hashing(monkeypatch): + assert _cache_key_in_spend_log(monkeypatch, None, None) == "Cache OFF" + + +def test_get_logging_payload_still_hashes_when_caching_is_on_and_no_preset_key_exists(monkeypatch): + class _RecordingCache(litellm.Cache): + def __init__(self) -> None: + pass + + def get_cache_key(self, **kwargs) -> str: + return "hashed-from-" + kwargs["model"] + + assert _cache_key_in_spend_log(monkeypatch, _RecordingCache(), None) == "hashed-from-gpt-4o-mini" + + _TRACE_ONLY_STANDARD_LOGGING: Final = cast( StandardLoggingPayload, { @@ -613,11 +655,11 @@ def test_sanitize_request_body_for_spend_logs_payload_circular_reference(): assert sanitized == {"b": {"a": {}}} # Should return empty dict for circular reference -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_vector_store_request_for_spend_logs_payload_store_prompts_true( mock_should_store, ): - # When _should_store_prompts_and_responses_in_spend_logs returns True + # When should_store_prompts_and_responses_in_spend_logs returns True mock_should_store.return_value = True # Sample vector store request metadata @@ -631,11 +673,11 @@ def test_get_vector_store_request_for_spend_logs_payload_store_prompts_true( assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"] == "sensitive information" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_vector_store_request_for_spend_logs_payload_store_prompts_false( mock_should_store, ): - # When _should_store_prompts_and_responses_in_spend_logs returns False + # When should_store_prompts_and_responses_in_spend_logs returns False mock_should_store.return_value = False # Sample vector store request metadata @@ -651,7 +693,7 @@ def test_get_vector_store_request_for_spend_logs_payload_store_prompts_false( assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["type"] == "text" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_vector_store_request_for_spend_logs_payload_null_input(mock_should_store): # When input is None mock_should_store.return_value = False @@ -659,7 +701,7 @@ def test_get_vector_store_request_for_spend_logs_payload_null_input(mock_should_ assert result is None -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_messages_for_spend_logs_realtime_returns_messages(mock_should_store): """ Test that _get_messages_for_spend_logs_payload returns messages @@ -686,7 +728,7 @@ def test_get_messages_for_spend_logs_realtime_returns_messages(mock_should_store assert parsed[1]["content"] == "What is the weather today?" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_messages_for_spend_logs_strips_null_bytes(mock_should_store): """Regression for PostgreSQL 22P05: NUL bytes must be stripped from messages.""" mock_should_store.return_value = True @@ -703,7 +745,7 @@ def test_get_messages_for_spend_logs_strips_null_bytes(mock_should_store): assert parsed[0]["content"] == "helloworld" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_messages_for_spend_logs_realtime_empty_when_disabled(mock_should_store): """ Test that _get_messages_for_spend_logs_payload returns '{}' for realtime calls @@ -721,7 +763,7 @@ def test_get_messages_for_spend_logs_realtime_empty_when_disabled(mock_should_st assert result == "{}" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_messages_for_spend_logs_non_realtime_returns_empty(mock_should_store): """ Test that _get_messages_for_spend_logs_payload returns '{}' for non-realtime @@ -739,7 +781,7 @@ def test_get_messages_for_spend_logs_non_realtime_returns_empty(mock_should_stor assert result == "{}" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_response_for_spend_logs_payload_truncates_large_base64(mock_should_store): from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB @@ -767,7 +809,7 @@ def test_get_response_for_spend_logs_payload_truncates_large_base64(mock_should_ assert parsed["data"][0]["other_field"] == "value" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_response_for_spend_logs_payload_strips_null_bytes(mock_should_store): """Regression for PostgreSQL 22P05: NUL bytes must be stripped from response.""" mock_should_store.return_value = True @@ -780,7 +822,7 @@ def test_get_response_for_spend_logs_payload_strips_null_bytes(mock_should_store assert json.loads(response_json)["content"] == "answerhere" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_response_for_spend_logs_payload_truncates_large_embedding( mock_should_store, ): @@ -835,7 +877,7 @@ def test_truncation_includes_db_safeguard_note(): ) -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_response_truncation_logs_info_message(mock_should_store): """ Test that when response is truncated before DB storage, an info log is emitted @@ -857,7 +899,7 @@ def test_response_truncation_logs_info_message(mock_should_store): assert "response was truncated" in log_msg -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_request_body_truncation_logs_info_message(mock_should_store): """ Test that when request body is truncated before DB storage, an info log is emitted. @@ -1520,7 +1562,7 @@ def test_get_logging_payload_handles_missing_overhead_gracefully(): ) -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_spend_logs_redacts_request_and_response_when_turn_off_message_logging_enabled( mock_should_store, ): @@ -1584,7 +1626,7 @@ def test_should_store_prompts_and_responses_in_spend_logs_case_insensitive_strin mock_get_secret_bool, ): """ - Test that _should_store_prompts_and_responses_in_spend_logs handles + Test that should_store_prompts_and_responses_in_spend_logs handles case-insensitive string values for store_prompts_in_spend_logs in general_settings. """ # Test case-insensitive string "true" variations @@ -1594,7 +1636,7 @@ def test_should_store_prompts_and_responses_in_spend_logs_case_insensitive_strin {"store_prompts_in_spend_logs": true_value}, ): mock_get_secret_bool.return_value = False # Ensure env var is False - result = _should_store_prompts_and_responses_in_spend_logs() + result = should_store_prompts_and_responses_in_spend_logs() assert result is True, f"Expected True for '{true_value}', got {result}" # Test boolean True @@ -1603,7 +1645,7 @@ def test_should_store_prompts_and_responses_in_spend_logs_case_insensitive_strin {"store_prompts_in_spend_logs": True}, ): mock_get_secret_bool.return_value = False - result = _should_store_prompts_and_responses_in_spend_logs() + result = should_store_prompts_and_responses_in_spend_logs() assert result is True, f"Expected True for boolean True, got {result}" # Test that non-true values fall back to environment variable @@ -1614,22 +1656,22 @@ def test_should_store_prompts_and_responses_in_spend_logs_case_insensitive_strin ): # When env var is True, should return True mock_get_secret_bool.return_value = True - result = _should_store_prompts_and_responses_in_spend_logs() + result = should_store_prompts_and_responses_in_spend_logs() assert result is True, f"Expected True (from env var) for '{false_value}', got {result}" # When env var is False, should return False mock_get_secret_bool.return_value = False - result = _should_store_prompts_and_responses_in_spend_logs() + result = should_store_prompts_and_responses_in_spend_logs() assert result is False, f"Expected False (from env var) for '{false_value}', got {result}" # Test when general_settings doesn't have the key at all with patch("litellm.proxy.proxy_server.general_settings", {}): mock_get_secret_bool.return_value = True - result = _should_store_prompts_and_responses_in_spend_logs() + result = should_store_prompts_and_responses_in_spend_logs() assert result is True, "Expected True (from env var) when key missing, got False" mock_get_secret_bool.return_value = False - result = _should_store_prompts_and_responses_in_spend_logs() + result = should_store_prompts_and_responses_in_spend_logs() assert result is False, "Expected False (from env var) when key missing, got True" @@ -1662,7 +1704,7 @@ def test_get_spend_logs_metadata_guardrail_info_fallback_from_metadata(): assert result["guardrail_information"] is None -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_redacts_all_prompt_carrying_fields_when_flag_false( mock_should_store, ): @@ -1698,7 +1740,7 @@ def test_sanitize_guardrail_information_redacts_all_prompt_carrying_fields_when_ assert entry["guardrail_action"] == "NONE" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_redacts_prompt_fields_when_flag_false( mock_should_store, ): @@ -1762,7 +1804,7 @@ def test_sanitize_guardrail_information_redacts_prompt_fields_when_flag_false( } -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_preserves_guardrail_usage_when_flag_false( mock_should_store, ): @@ -1794,7 +1836,7 @@ def test_sanitize_guardrail_information_preserves_guardrail_usage_when_flag_fals assert entry["guardrail_usage"] == {"topicPolicyUnits": 1, "contentPolicyUnits": 1, "wordPolicyUnits": 0} -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_passthrough_when_flag_true( mock_should_store, ): @@ -1817,13 +1859,13 @@ def test_sanitize_guardrail_information_passthrough_when_flag_true( assert result == guardrail_info -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_none_passthrough(mock_should_store): mock_should_store.return_value = False assert _sanitize_guardrail_information_for_spend_logs(None) is None -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_normalizes_bare_dict_input(mock_should_store): """ Regression: xecguard (xecguard.py:246) assigns a bare dict to @@ -1855,7 +1897,7 @@ def test_sanitize_guardrail_information_normalizes_bare_dict_input(mock_should_s assert entry["start_time"] == 1.0 -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_drops_non_dict_items_in_list(mock_should_store): """ A stray non-dict item in the list (e.g. from a buggy caller that @@ -1874,7 +1916,7 @@ def test_sanitize_guardrail_information_drops_non_dict_items_in_list(mock_should assert result == [{"guardrail_name": "x", "guardrail_response": REDACTED_BY_LITELM_STRING}] -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_preserves_absent_prompt_fields(mock_should_store): """ Entries that never carried guardrail_request or guardrail_response must @@ -2290,7 +2332,7 @@ def test_sanitize_request_body_strips_secret_fields(): assert sanitized["messages"] == [{"role": "user", "content": "hi"}] -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_proxy_server_request_payload_excludes_secret_fields(mock_should_store): """ End-to-end test: when the proxy_server_request body contains @@ -2370,7 +2412,7 @@ def test_redact_prompt_leaks_empty_string(): assert _redact_prompt_leaks_in_error_string("") == "" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_redacts_when_not_storing_prompts( mock_should_store, ): @@ -2398,7 +2440,7 @@ def test_sanitize_error_information_redacts_when_not_storing_prompts( assert sanitized["llm_provider"] == "openai" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_skips_redaction_when_storing_prompts( mock_should_store, ): @@ -2420,7 +2462,7 @@ def test_sanitize_error_information_skips_redaction_when_storing_prompts( assert REDACTED_BY_LITELM_STRING not in sanitized["error_message"] -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_caps_size_regardless_of_prompt_flag( mock_should_store, ): @@ -2451,7 +2493,7 @@ def test_sanitize_error_information_none_passthrough(): assert _sanitize_error_information_for_spend_logs(None) is None -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_reproduces_lit_2992(mock_should_store): # Mirrors the reproduced row body from LIT-2992 — a RateLimitError whose # message embeds 178 pydantic validation errors, each carrying a full @@ -2536,7 +2578,7 @@ def test_redact_prompt_leaks_handles_unterminated_value(): assert REDACTED_BY_LITELM_STRING in redacted -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_redacts_traceback_when_not_storing_prompts( mock_should_store, ): @@ -2568,7 +2610,7 @@ def test_sanitize_error_information_redacts_traceback_when_not_storing_prompts( assert "ValueError: invalid request" in sanitized["traceback"] -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_skips_traceback_redaction_when_storing_prompts( mock_should_store, ): @@ -2676,7 +2718,7 @@ def test_redact_prompt_leaks_combined_quoted_key_and_pydantic_assignment(): assert redacted.count(REDACTED_BY_LITELM_STRING) >= 2 -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_redacts_pydantic_assignment_form( mock_should_store, ): @@ -3272,7 +3314,7 @@ def test_get_logging_payload_hashes_bearer_prefixed_api_key(): ) -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_preserves_headroom_compression_token_stats( mock_should_store, ): diff --git a/tests/test_litellm/proxy/test_collector.py b/tests/test_litellm/proxy/test_collector.py new file mode 100644 index 00000000000..ac3e1f5e566 --- /dev/null +++ b/tests/test_litellm/proxy/test_collector.py @@ -0,0 +1,229 @@ +import asyncio +import logging +from collections.abc import Callable, Iterator +from pathlib import Path +from typing import Final + +import pytest +import uvloop + +from litellm._logging import verbose_logger, verbose_proxy_logger, verbose_router_logger +from litellm.proxy.collector import ( + SpendEventConsumer, + address_argument, + apply_log_level, + pod_pgbouncer_database_url, +) +from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings +from litellm.proxy.spend_tracking.spend_event_producer import ( + AddressError, + SpendEventProducer, + TcpAddress, + UnixAddress, + open_collector_connection, +) + + +class _Handler: + def __init__(self, fail_on: bytes | None = None) -> None: + self.lines: list[bytes] = [] # mutable-ok: test double records the events the consumer handed over + self._fail_on = fail_on + + async def __call__(self, line: bytes) -> None: + if line == self._fail_on: + raise RuntimeError("pipeline failed") + self.lines.append(line) + + +async def _no_fallback(line: bytes) -> None: + raise AssertionError(f"unexpected fallback for {line!r}") + + +class _Fallback: + def __init__(self) -> None: + self.lines: list[bytes] = [] # mutable-ok: test double records the events run in-process + + async def __call__(self, line: bytes) -> None: + self.lines.append(line) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("transport", ["unix", "tcp"]) +async def test_consumer_handles_each_producer_line_once_in_order(tmp_path: Path, transport: str): + handler: Final = _Handler(fail_on=b"event-3\n") + consumer: Final = SpendEventConsumer(handler) + server: Final = await consumer.serve( + UnixAddress(path=str(tmp_path / "spend.sock")) if transport == "unix" else TcpAddress("127.0.0.1", 0) + ) + address: Final = ( + UnixAddress(path=str(tmp_path / "spend.sock")) + if transport == "unix" + else TcpAddress("127.0.0.1", server.sockets[0].getsockname()[1]) + ) + producer: Final = SpendEventProducer( + address=address, on_unavailable="fallback", buffer_size=100, connect_timeout=1.0, fallback=_no_fallback + ) + for i in range(6): + await producer.publish(f"event-{i}\n".encode()) + await producer.close(drain_timeout=5.0) + + server.close() + assert await consumer.drain(timeout=5.0) == 0 + assert handler.lines == [f"event-{i}\n".encode() for i in range(6) if i != 3] + assert (consumer.received, consumer.handled, consumer.failed) == (6, 5, 1) + + +@pytest.mark.asyncio +async def test_consumer_discards_a_truncated_trailing_event(tmp_path: Path): + handler: Final = _Handler() + consumer: Final = SpendEventConsumer(handler) + address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + server: Final = await consumer.serve(address) + _, writer = await open_collector_connection(address, timeout=1.0) + writer.write(b"whole\npartial-without-newline") + await writer.drain() + writer.close() + await writer.wait_closed() + await asyncio.sleep(0.05) + + server.close() + assert await consumer.drain(timeout=5.0) == 0 + assert handler.lines == [b"whole\n"] + assert consumer.received == 1 + + +@pytest.mark.asyncio +async def test_drain_reports_producers_still_connected_after_the_timeout(tmp_path: Path): + consumer: Final = SpendEventConsumer(_Handler()) + address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + server: Final = await consumer.serve(address) + _, writer = await open_collector_connection(address, timeout=1.0) + await asyncio.sleep(0.05) + + server.close() + assert await consumer.drain(timeout=0.1) == 1 + writer.close() + await writer.wait_closed() + assert await consumer.drain(timeout=5.0) == 0 + + +@pytest.mark.asyncio +async def test_graceful_stop_hands_the_producer_over_to_its_fallback_without_losing_events(tmp_path: Path): + handler: Final = _Handler() + fallback: Final = _Fallback() + consumer: Final = SpendEventConsumer(handler) + address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + server: Final = await consumer.serve(address) + producer: Final = SpendEventProducer( + address=address, on_unavailable="fallback", buffer_size=100, connect_timeout=1.0, fallback=fallback + ) + await producer.publish(b"event-1\n") + await asyncio.sleep(0.05) + + server.close() + draining: Final = asyncio.ensure_future(consumer.drain(timeout=5.0)) + await asyncio.sleep(0.05) + await producer.publish(b"event-2\n") + await producer.close(drain_timeout=5.0) + + assert await draining == 0 + assert handler.lines == [b"event-1\n"] + assert fallback.lines == [b"event-2\n"] + assert (producer.stats().sent, producer.stats().fallback) == (1, 1) + + +@pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"]) +def test_drain_still_hands_over_live_producers_when_another_connection_already_died( + tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] +): + """A transport the loop force-closed under a busy handler must not abort the half-close of the others.""" + + async def scenario() -> tuple[int, list[bytes]]: + release: Final = asyncio.Event() + + async def slow_handler(line: bytes) -> None: + await release.wait() + + consumer: Final = SpendEventConsumer(slow_handler) + address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + server: Final = await consumer.serve(address) + _, dead = await open_collector_connection(address, timeout=1.0) + dead.write(b"stuck\n") + await dead.drain() + await asyncio.sleep(0.05) + for connection in consumer._open_connections: # pyright: ignore[reportPrivateUsage] # force-close like uvloop does on a socket error + connection.transport.close() + dead.close() + fallback: Final = _Fallback() + producer: Final = SpendEventProducer( + address=address, on_unavailable="fallback", buffer_size=100, connect_timeout=1.0, fallback=fallback + ) + await producer.publish(b"event-1\n") + await asyncio.sleep(0.05) + + server.close() + draining: Final = asyncio.ensure_future(consumer.drain(timeout=0.5)) + await asyncio.sleep(0.05) + await producer.publish(b"event-2\n") + await producer.close(drain_timeout=5.0) + still_open: Final = await draining + release.set() + await asyncio.sleep(0.05) + return still_open, fallback.lines + + with asyncio.Runner(loop_factory=loop_factory) as runner: + still_open, fallback_lines = runner.run(scenario()) + + assert still_open == 2 + assert fallback_lines == [b"event-2\n"] + + +def test_address_argument(): + assert address_argument((), default="unix:///tmp/x.sock") == "unix:///tmp/x.sock" + assert address_argument(("--address", "tcp://127.0.0.1:4100"), default="unix:///tmp/x.sock") == ( + "tcp://127.0.0.1:4100" + ) + assert isinstance(address_argument(("--listen", "x"), default="unix:///tmp/x.sock"), AddressError) + + +def test_pod_pgbouncer_database_url_points_at_the_proxy_containers_pooler(): + """With pgbouncer on, the sidecar must not open its own upstream connections but share the pod's pooler.""" + upstream: Final = "postgresql://u:p@db.internal:5432/litellm?schema=public" + environ: Final = {"DATABASE_URL": upstream} + assert pod_pgbouncer_database_url(PgBouncerSettings(enabled=False), environ, token_auth=False) is None + assert ( + pod_pgbouncer_database_url(PgBouncerSettings(enabled=True, port=6543), environ, token_auth=False) + == "postgresql://u:p@127.0.0.1:6543/litellm?schema=public&pgbouncer=true" + ) + assert isinstance(pod_pgbouncer_database_url(PgBouncerSettings(enabled=True), {}, token_auth=False), PgBouncerError) + + +def test_pod_pgbouncer_database_url_goes_direct_under_token_auth(): + """The proxy's pgbouncer only knows the token that container minted, so the sidecar must mint its own upstream.""" + iam_upstream: Final = "postgresql://u@db.internal:5432/litellm?schema=public" + assert ( + pod_pgbouncer_database_url(PgBouncerSettings(enabled=True), {"DATABASE_URL": iam_upstream}, token_auth=True) + is None + ) + assert pod_pgbouncer_database_url(PgBouncerSettings(enabled=True), {}, token_auth=True) is None + + +@pytest.fixture +def restore_log_levels() -> Iterator[None]: + loggers: Final = (verbose_logger, verbose_router_logger, verbose_proxy_logger) + levels: Final = tuple(logger.level for logger in loggers) + yield + for logger, level in zip(loggers, levels, strict=True): + logger.setLevel(level) + + +@pytest.mark.usefixtures("restore_log_levels") +@pytest.mark.parametrize( + ("litellm_log", "expected"), + [("DEBUG", logging.DEBUG), ("info", logging.INFO), (None, logging.WARNING), ("loud", logging.WARNING)], +) +def test_apply_log_level_mirrors_the_proxy_env_contract(litellm_log: str | None, expected: int): + verbose_proxy_logger.setLevel(logging.WARNING) + apply_log_level(litellm_log) + assert verbose_proxy_logger.isEnabledFor(expected) + assert not verbose_proxy_logger.isEnabledFor(expected - 10) From 960fc4b114815cd84ac08f2f98d675286e1c9d67 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:16:15 -0700 Subject: [PATCH 102/126] feat(newrelic): export team max and remaining budget gauges to the Metric API (#40542) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integrations/newrelic/newrelic_metrics.py | 39 +++++++- litellm/types/integrations/newrelic.py | 19 +++- .../newrelic/test_newrelic_metrics.py | 92 ++++++++++++++++++- 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/newrelic/newrelic_metrics.py b/litellm/integrations/newrelic/newrelic_metrics.py index 25dbfc2bdb2..da952b78d3f 100644 --- a/litellm/integrations/newrelic/newrelic_metrics.py +++ b/litellm/integrations/newrelic/newrelic_metrics.py @@ -5,8 +5,9 @@ NR Reference API: https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-ap `async_log_success_event` / `async_log_failure_event` queue one record per request; at flush the queue is aggregated by (team, model group, model, provider, status) -into count/summary metrics. `interval.ms` is the real window between flushes, -computed at flush time. +into count/summary metrics, plus one max/remaining budget gauge pair per team +taken from the team's latest record. `interval.ms` is the real window between +flushes, computed at flush time. Team-scoped by construction: the ingest key is injected explicitly and there is deliberately no environment-variable fallback, so a team's metrics are never sent @@ -47,11 +48,14 @@ from litellm.types.integrations.newrelic import ( NEWRELIC_METRIC_PROMPT_TOKENS, NEWRELIC_METRIC_REQUEST_DURATION_MS, NEWRELIC_METRIC_REQUESTS, + NEWRELIC_METRIC_TEAM_MAX_BUDGET, + NEWRELIC_METRIC_TEAM_REMAINING_BUDGET, NEWRELIC_METRIC_TOTAL_TOKENS, NEWRELIC_METRICS_MAX_BATCH_SIZE, NEWRELIC_METRICS_MAX_DRAIN_PASSES, NEWRELIC_METRICS_MAX_RETRY_QUEUE_SIZE, NewRelicCountMetric, + NewRelicGaugeMetric, NewRelicMetric, NewRelicMetricCommon, NewRelicMetricEnvelope, @@ -98,6 +102,8 @@ def _metric_record_from_payload(standard_logging_object: StandardLoggingPayload) completion_tokens=int(standard_logging_object.get("completion_tokens") or 0), total_tokens=int(standard_logging_object.get("total_tokens") or 0), duration_ms=float(standard_logging_object.get("response_time") or 0.0) * 1000.0, + team_max_budget=metadata.get("user_api_key_team_max_budget") if metadata else None, + team_spend=metadata.get("user_api_key_team_spend") if metadata else None, ) @@ -140,6 +146,33 @@ def _bucket_metrics(bucket_records: tuple[NewRelicMetricRecord, ...]) -> tuple[N return (*count_metrics, summary_metric) +def _team_budget_gauges(record: NewRelicMetricRecord) -> tuple[NewRelicMetric, ...]: + team_max_budget: Final = record.team_max_budget + if team_max_budget is None: + return () + attributes: Final[Mapping[str, str]] = { # mutable-ok: JSON leaf; safe_dumps stringifies MappingProxyType + key: value[:NEWRELIC_METRIC_ATTRIBUTE_MAX_LEN] + for key, value in (("team_id", record.team_id), ("team_alias", record.team_alias)) + if value + } + remaining_budget: Final = team_max_budget - (record.team_spend or 0.0) - record.response_cost + return ( + NewRelicGaugeMetric( + name=NEWRELIC_METRIC_TEAM_MAX_BUDGET, type="gauge", value=team_max_budget, attributes=attributes + ), + NewRelicGaugeMetric( + name=NEWRELIC_METRIC_TEAM_REMAINING_BUDGET, type="gauge", value=remaining_budget, attributes=attributes + ), + ) + + +def _team_budget_metrics(records: tuple[NewRelicMetricRecord, ...]) -> tuple[NewRelicMetric, ...]: + latest_by_team: Final[Mapping[str, NewRelicMetricRecord]] = MappingProxyType( + {record.team_id: record for record in records if record.team_id} + ) + return tuple(gauge for record in latest_by_team.values() for gauge in _team_budget_gauges(record)) + + def build_metric_payload( records: tuple[NewRelicMetricRecord, ...], *, @@ -158,7 +191,7 @@ def build_metric_payload( "timestamp": int(window_start * 1000), "interval.ms": interval_ms, } - return (NewRelicMetricEnvelope(common=common, metrics=metrics),) + return (NewRelicMetricEnvelope(common=common, metrics=(*metrics, *_team_budget_metrics(records))),) class NewRelicMetricsLogger(CustomBatchLogger): diff --git a/litellm/types/integrations/newrelic.py b/litellm/types/integrations/newrelic.py index 36e4d02c2a8..b5905ad0b93 100644 --- a/litellm/types/integrations/newrelic.py +++ b/litellm/types/integrations/newrelic.py @@ -27,9 +27,9 @@ NEWRELIC_METRIC_ENDPOINT_BY_REGION: Final[Mapping[str, str]] = MappingProxyType( NEWRELIC_DEFAULT_REGION: Final = "us" #: Metric API caps a payload at 2000 data points / 1MB compressed; each queued -#: record expands to at most 6 metrics, so cap the per-flush record count well -#: below that. -NEWRELIC_METRICS_MAX_BATCH_SIZE: Final = 250 +#: record expands to at most 6 bucket metrics plus 2 team budget gauges (8), so cap +#: the per-flush record count well below 2000 / 8. +NEWRELIC_METRICS_MAX_BATCH_SIZE: Final = 200 #: Hard cap on records retained across failed flushes (5xx/network requeue). #: Beyond this the oldest records are dropped. @@ -48,6 +48,8 @@ NEWRELIC_METRIC_PROMPT_TOKENS: Final = "litellm.tokens.prompt" NEWRELIC_METRIC_COMPLETION_TOKENS: Final = "litellm.tokens.completion" NEWRELIC_METRIC_TOTAL_TOKENS: Final = "litellm.tokens.total" NEWRELIC_METRIC_REQUEST_DURATION_MS: Final = "litellm.request.duration_ms" +NEWRELIC_METRIC_TEAM_MAX_BUDGET: Final = "litellm.team.max_budget" +NEWRELIC_METRIC_TEAM_REMAINING_BUDGET: Final = "litellm.team.remaining_budget" class NewRelicSummaryValue(TypedDict): @@ -66,6 +68,13 @@ class NewRelicCountMetric(TypedDict): attributes: ReadOnly[Mapping[str, str]] +class NewRelicGaugeMetric(TypedDict): + name: ReadOnly[str] + type: ReadOnly[Literal["gauge"]] + value: ReadOnly[float] + attributes: ReadOnly[Mapping[str, str]] + + class NewRelicSummaryMetric(TypedDict): name: ReadOnly[str] type: ReadOnly[Literal["summary"]] @@ -73,7 +82,7 @@ class NewRelicSummaryMetric(TypedDict): attributes: ReadOnly[Mapping[str, str]] -NewRelicMetric = NewRelicCountMetric | NewRelicSummaryMetric +NewRelicMetric = NewRelicCountMetric | NewRelicGaugeMetric | NewRelicSummaryMetric #: ``interval.ms`` has a dot in it, so the functional TypedDict form is required. @@ -108,6 +117,8 @@ class NewRelicMetricRecord: completion_tokens: int total_tokens: int duration_ms: float + team_max_budget: float | None = None + team_spend: float | None = None @property def bucket_key(self) -> tuple[str, str, str, str, str, str]: diff --git a/tests/test_litellm/integrations/newrelic/test_newrelic_metrics.py b/tests/test_litellm/integrations/newrelic/test_newrelic_metrics.py index 9c75e0b0a47..a9074e89806 100644 --- a/tests/test_litellm/integrations/newrelic/test_newrelic_metrics.py +++ b/tests/test_litellm/integrations/newrelic/test_newrelic_metrics.py @@ -24,6 +24,8 @@ from litellm.types.integrations.newrelic import ( NEWRELIC_METRIC_PROMPT_TOKENS, NEWRELIC_METRIC_REQUEST_DURATION_MS, NEWRELIC_METRIC_REQUESTS, + NEWRELIC_METRIC_TEAM_MAX_BUDGET, + NEWRELIC_METRIC_TEAM_REMAINING_BUDGET, NEWRELIC_METRIC_TOTAL_TOKENS, NewRelicMetricRecord, ) @@ -40,6 +42,8 @@ def _record( completion_tokens=20, total_tokens=30, duration_ms=100.0, + team_max_budget=None, + team_spend=None, ) -> NewRelicMetricRecord: return NewRelicMetricRecord( team_id=team_id, @@ -53,12 +57,28 @@ def _record( completion_tokens=completion_tokens, total_tokens=total_tokens, duration_ms=duration_ms, + team_max_budget=team_max_budget, + team_spend=team_spend, ) -def _standard_logging_object(team_id="team-a", response_cost=0.25) -> dict: +def _standard_logging_object( + team_id="team-a", response_cost=0.25, team_max_budget: float | None = None, team_spend: float | None = None +) -> dict: + budget_metadata = { + key: value + for key, value in ( + ("user_api_key_team_max_budget", team_max_budget), + ("user_api_key_team_spend", team_spend), + ) + if value is not None + } return { - "metadata": {"user_api_key_team_id": team_id, "user_api_key_team_alias": f"{team_id}-alias"}, + "metadata": { + "user_api_key_team_id": team_id, + "user_api_key_team_alias": f"{team_id}-alias", + **budget_metadata, + }, "model_group": "gpt-4o-group", "model": "gpt-4o", "custom_llm_provider": "openai", @@ -204,6 +224,74 @@ class TestBuildMetricPayload: assert "model_group" not in attributes +class TestTeamBudgetGauges: + def test_latest_record_per_team_drives_one_gauge_pair(self): + records = ( + _record(team_id="team-a", model="gpt-4o", response_cost=0.5, team_max_budget=100.0, team_spend=10.0), + _record(team_id="team-a", model="claude-4", response_cost=2.0, team_max_budget=100.0, team_spend=10.5), + _record(team_id="team-b", response_cost=1.0, team_max_budget=None, team_spend=3.0), + _record(team_id="", team_alias="", response_cost=1.0, team_max_budget=50.0, team_spend=1.0), + ) + payload = build_metric_payload(records, window_start=1_000.0, now=1_005.0) + + max_budget_gauges = _metrics_by_name(payload, NEWRELIC_METRIC_TEAM_MAX_BUDGET) + remaining_gauges = _metrics_by_name(payload, NEWRELIC_METRIC_TEAM_REMAINING_BUDGET) + assert [(m["type"], m["value"], m["attributes"]) for m in max_budget_gauges] == [ + ("gauge", 100.0, {"team_id": "team-a", "team_alias": "team-a-alias"}) + ] + assert [(m["type"], m["attributes"]) for m in remaining_gauges] == [ + ("gauge", {"team_id": "team-a", "team_alias": "team-a-alias"}) + ] + assert remaining_gauges[0]["value"] == pytest.approx(100.0 - 10.5 - 2.0) + assert len(_metrics_by_name(payload, NEWRELIC_METRIC_COST_USD)) == 4 + + def test_missing_team_spend_counts_only_this_request(self): + payload = build_metric_payload( + (_record(response_cost=0.25, team_max_budget=10.0, team_spend=None),), window_start=1_000.0, now=1_005.0 + ) + + assert _metrics_by_name(payload, NEWRELIC_METRIC_TEAM_REMAINING_BUDGET)[0]["value"] == pytest.approx(9.75) + + @pytest.mark.asyncio + async def test_budget_gauges_reach_the_metric_api_from_standard_logging_metadata(self): + logger = _make_logger() + logger.async_client.post = AsyncMock(return_value=_response(202)) + slo = _standard_logging_object(response_cost=0.25, team_max_budget=20.0, team_spend=4.5) + + await logger.async_log_success_event( + kwargs={"standard_logging_object": slo}, response_obj={}, start_time=None, end_time=None + ) + await logger.flush_queue() + + body = json.loads(gzip.decompress(logger.async_client.post.await_args.kwargs["data"]).decode("utf-8")) + by_name = {m["name"]: m for m in body[0]["metrics"]} + assert by_name[NEWRELIC_METRIC_TEAM_MAX_BUDGET] == { + "name": NEWRELIC_METRIC_TEAM_MAX_BUDGET, + "type": "gauge", + "value": 20.0, + "attributes": {"team_id": "team-a", "team_alias": "team-a-alias"}, + } + assert by_name[NEWRELIC_METRIC_TEAM_REMAINING_BUDGET]["type"] == "gauge" + assert by_name[NEWRELIC_METRIC_TEAM_REMAINING_BUDGET]["value"] == pytest.approx(15.25) + assert by_name[NEWRELIC_METRIC_COST_USD]["value"] == 0.25 + + @pytest.mark.asyncio + async def test_no_budget_metadata_sends_no_gauges(self): + logger = _make_logger() + logger.async_client.post = AsyncMock(return_value=_response(202)) + + await logger.async_log_success_event( + kwargs={"standard_logging_object": _standard_logging_object()}, + response_obj={}, + start_time=None, + end_time=None, + ) + await logger.flush_queue() + + body = json.loads(gzip.decompress(logger.async_client.post.await_args.kwargs["data"]).decode("utf-8")) + assert {m["type"] for m in body[0]["metrics"]} == {"count", "summary"} + + class TestQueueAndFlush: @pytest.mark.asyncio async def test_log_event_queues_record_from_standard_logging_object(self): From 1f7c4d6784411efa2c9622aa6ea7dcf3e0320e89 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 10 Sep 2026 17:18:56 -0700 Subject: [PATCH 103/126] test: fix stale completion response fixtures --- tests/llm_translation/test_azure_o_series.py | 8 ++++++++ tests/llm_translation/test_azure_openai.py | 5 +++++ tests/llm_translation/test_openai.py | 6 ++++++ tests/local_testing/test_completion.py | 9 +++++++-- 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/llm_translation/test_azure_o_series.py b/tests/llm_translation/test_azure_o_series.py index 1a2d672af71..ce7e614cbe2 100644 --- a/tests/llm_translation/test_azure_o_series.py +++ b/tests/llm_translation/test_azure_o_series.py @@ -159,15 +159,23 @@ def test_azure_o_series_routing(): def test_openai_o_series_max_retries_0(mock_get_openai_client): import litellm + mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.headers = {} + mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.parse.return_value = ( + ModelResponse(choices=[{"message": {"role": "assistant", "content": "Hello"}}]) + ) litellm.set_verbose = True response = litellm.completion( model="azure/o1-preview", messages=[{"role": "user", "content": "hi"}], max_retries=0, + api_key="fake-key", + api_base="https://fake-azure.openai.azure.com", + api_version="2024-10-21", ) mock_get_openai_client.assert_called_once() assert mock_get_openai_client.call_args.kwargs["max_retries"] == 0 + assert response.choices[0].message.content == "Hello" @pytest.mark.asyncio diff --git a/tests/llm_translation/test_azure_openai.py b/tests/llm_translation/test_azure_openai.py index 0fa72b45ed8..e6528e77749 100644 --- a/tests/llm_translation/test_azure_openai.py +++ b/tests/llm_translation/test_azure_openai.py @@ -335,6 +335,10 @@ def test_azure_gpt_4o_with_tool_call_and_response_format(api_version): ] with patch.object(client.chat.completions.with_raw_response, "create") as mock_post: + mock_post.return_value.headers = {} + mock_post.return_value.parse.return_value = litellm.ModelResponse( + choices=[{"message": {"role": "assistant", "content": InvestigationOutput().model_dump_json()}}] + ) response = litellm.completion( model="azure/gpt-4.1-mini", messages=[ @@ -362,6 +366,7 @@ def test_azure_gpt_4o_with_tool_call_and_response_format(api_version): assert "response_format" in mock_post.call_args.kwargs else: assert "response_format" not in mock_post.call_args.kwargs + assert response.choices[0].message.content == InvestigationOutput().model_dump_json() def test_map_openai_params(): diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 2b9abdec5d0..af4ba85d58e 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -292,15 +292,21 @@ class TestOpenAIChatCompletion(BaseLLMChatTest): def test_openai_max_retries_0(mock_get_openai_client): import litellm + mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.headers = {} + mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.parse.return_value = ( + ModelResponse(choices=[{"message": {"role": "assistant", "content": "Hello"}}]) + ) litellm.set_verbose = True response = litellm.completion( model="gpt-4o-mini", messages=[{"role": "user", "content": "hi"}], max_retries=0, + api_key="fake-key", ) mock_get_openai_client.assert_called_once() assert mock_get_openai_client.call_args.kwargs["max_retries"] == 0 + assert response.choices[0].message.content == "Hello" @patch("litellm.main.openai_chat_completions._get_openai_client") diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index f8f23ea015a..43ed57f63af 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -3999,10 +3999,14 @@ def test_completion_novita_ai(): openai_client = OpenAI(api_key="fake-key") with patch.object( - openai_client.chat.completions, "create", new=MagicMock() + openai_client.chat.completions.with_raw_response, "create" ) as mock_call: + mock_call.return_value.headers = {} + mock_call.return_value.parse.return_value = litellm.ModelResponse( + choices=[{"message": {"role": "assistant", "content": "Hello"}}] + ) try: - completion( + response = completion( model="novita/meta-llama/llama-3.3-70b-instruct", messages=messages, client=openai_client, @@ -4010,6 +4014,7 @@ def test_completion_novita_ai(): ) mock_call.assert_called_once() + assert response.choices[0].message.content == "Hello" # Verify model is passed correctly assert ( From c7a41c35d5cdf2fc2d270c6fee6c793729eeee06 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 00:48:25 +0000 Subject: [PATCH 104/126] perf(mock): emit admission-time usage chunk on streaming mock_response (#40637) * perf(mock): emit admission-time usage chunk on streaming mock_response Streaming mock_response chunks carried no usage, so the chunk builder re-tokenized the whole prompt in Python after the stream ended even when budget reservation had already counted it at admission. The mock streaming generators now yield a final usage-only chunk carrying the admission prompt count (same completion count as the non-streaming path). Without an admission count the old tokenizer fallback stays. * fix(mock): type the mock stream generators and keep the usage chunk on the content stream id Review follow-up: the usage-only chunk was built with a fresh id, so CustomStreamWrapper switched response_id for the finish-reason and usage chunks. It now copies the content stream id. The generators also get full parameter and return annotations. --------- Co-authored-by: yassin --- litellm/main.py | 4 +- litellm/utils.py | 35 +++- .../test_streaming_chunk_builder_utils.py | 49 ++++++ tests/test_litellm/test_main.py | 158 ++++++++++++++++++ tests/test_litellm/test_utils.py | 90 ++++++++++ 5 files changed, 329 insertions(+), 7 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 1a4beb787bc..10fb32828f1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -939,7 +939,7 @@ def mock_completion( if kwargs.get("acompletion", False) is True: return CustomStreamWrapper( completion_stream=async_mock_completion_streaming_obj( - model_response, mock_response=mock_response, model=model, n=n + model_response, mock_response=mock_response, model=model, n=n, prompt_tokens=prompt_tokens ), model=model, custom_llm_provider="openai", @@ -947,7 +947,7 @@ def mock_completion( ) return CustomStreamWrapper( completion_stream=mock_completion_streaming_obj( - model_response, mock_response=mock_response, model=model, n=n + model_response, mock_response=mock_response, model=model, n=n, prompt_tokens=prompt_tokens ), model=model, custom_llm_provider="openai", diff --git a/litellm/utils.py b/litellm/utils.py index aced4c9b312..b93938a480e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -66,6 +66,7 @@ from litellm.constants import ( DEFAULT_EMBEDDING_PARAM_VALUES, DEFAULT_MAX_LRU_CACHE_SIZE, DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT, + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, DEFAULT_TRIM_RATIO, FUNCTION_DEFINITION_TOKEN_COUNT, @@ -278,7 +279,7 @@ except (ImportError, AttributeError, TypeError): # Convert to str (if necessary) claude_json_str = json.dumps(json_data) import importlib.metadata -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast, get_args from litellm import utils as litellm_utils @@ -7001,7 +7002,26 @@ class TextCompletionStreamWrapper: raise StopAsyncIteration -def mock_completion_streaming_obj(model_response, mock_response, model, n: int | None = None): +def mock_stream_usage_chunk(model_response: ModelResponseStream, model: str, prompt_tokens: int) -> ModelResponseStream: + return ModelResponseStream( + id=model_response.id, + choices=[], # mutable-ok: ModelResponseStream only treats a list as explicit choices, a tuple gets a default choice + model=model, + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, + total_tokens=prompt_tokens + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, + ), + ) + + +def mock_completion_streaming_obj( + model_response: ModelResponseStream, + mock_response: str | MockException | ModelResponseStream, + model: str, + n: int | None = None, + prompt_tokens: int | None = None, +) -> Iterator[ModelResponseStream]: if isinstance(mock_response, litellm.MockException): raise mock_response if isinstance(mock_response, ModelResponseStream): @@ -7021,14 +7041,17 @@ def mock_completion_streaming_obj(model_response, mock_response, model, n: int | _all_choices.append(_streaming_choice) model_response.choices = _all_choices yield model_response + if prompt_tokens is not None: + yield mock_stream_usage_chunk(model_response, model=model, prompt_tokens=prompt_tokens) async def async_mock_completion_streaming_obj( - model_response, + model_response: ModelResponseStream, mock_response: str | MockException | ModelResponseStream, - model, + model: str, n: int | None = None, -): + prompt_tokens: int | None = None, +) -> AsyncIterator[ModelResponseStream]: if isinstance(mock_response, litellm.MockException): raise mock_response if isinstance(mock_response, ModelResponseStream): @@ -7048,6 +7071,8 @@ async def async_mock_completion_streaming_obj( _all_choices.append(_streaming_choice) model_response.choices = _all_choices yield model_response + if prompt_tokens is not None: + yield mock_stream_usage_chunk(model_response, model=model, prompt_tokens=prompt_tokens) ########## Reading Config File ############################ diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 626b8a63b20..efe4209c1c9 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1554,3 +1554,52 @@ def test_stream_chunk_builder_reads_role_from_first_frame_with_choices() -> None assert response is not None assert response.choices[0].message.role == "user" assert response.choices[0].message.content == "Hi" + + +def _fail_prompt_token_count() -> int: + raise AssertionError("prompt tokens must come from the usage chunk, not the tokenizer") + + +def test_calculate_usage_reads_prompt_tokens_from_mock_stream_usage_chunk_without_tokenizer_fallback() -> None: + from litellm.utils import mock_completion_streaming_obj + + chunks: Final = list( + mock_completion_streaming_obj( + ModelResponseStream(model="gpt-5.4-mini"), + mock_response="ok", + model="gpt-5.4-mini", + prompt_tokens=51234, + ) + ) + assert chunks[-1].choices == [] + + usage: Final = ChunkProcessor(chunks=chunks).calculate_usage( + chunks=chunks, + model="gpt-5.4-mini", + completion_output="ok", + count_prompt_tokens=_fail_prompt_token_count, + ) + + assert usage.prompt_tokens == 51234 + assert usage.completion_tokens == chunks[-1].usage.completion_tokens + assert usage.total_tokens == 51234 + usage.completion_tokens + + +def test_calculate_usage_falls_back_to_prompt_counter_when_mock_stream_has_no_admission_count() -> None: + from litellm.utils import mock_completion_streaming_obj + + chunks: Final = list( + mock_completion_streaming_obj( + ModelResponseStream(model="gpt-5.4-mini"), mock_response="ok", model="gpt-5.4-mini" + ) + ) + assert all(chunk.choices for chunk in chunks) + + usage: Final = ChunkProcessor(chunks=chunks).calculate_usage( + chunks=chunks, + model="gpt-5.4-mini", + completion_output="ok", + count_prompt_tokens=lambda: 77, + ) + + assert usage.prompt_tokens == 77 diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index f8841af9750..a36ca229981 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -2421,6 +2421,164 @@ def test_mock_completion_usage_falls_back_to_default_without_admission_count(): assert response.usage.prompt_tokens == litellm_main.DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT +_ADMISSION_INPUT_TOKENS: Final = 51234 +_ADMISSION_METADATA: Final = { + "user_api_key_budget_reservation": {"reserved_cost": 1.0, "input_tokens": _ADMISSION_INPUT_TOKENS} +} +_MOCK_STREAM_MESSAGES: Final = [{"role": "user", "content": "hello " * 200}] +_STREAM_CHUNK_BUILDER_TOKEN_COUNTER: Final = "litellm.litellm_core_utils.streaming_chunk_builder_utils.token_counter" + + +def _prompt_token_counter_calls(token_counter: MagicMock) -> list[object]: + return [call for call in token_counter.call_args_list if call.kwargs.get("messages") is not None] + + +def _client_usage_chunks(chunks: list[ModelResponseStream]) -> list[Usage]: + return [chunk.usage for chunk in chunks if getattr(chunk, "usage", None) is not None] + + +@pytest.mark.parametrize("n", (None, 2)) +def test_mock_completion_stream_usage_reports_admission_input_tokens_without_tokenizer_fallback(n: int | None): + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + chunks = list( + litellm.completion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + n=n, + stream_options={"include_usage": True}, + metadata=_ADMISSION_METADATA, + ) + ) + + usage_chunks = _client_usage_chunks(chunks) + assert len(usage_chunks) == 1 + assert usage_chunks[0].prompt_tokens == _ADMISSION_INPUT_TOKENS + assert usage_chunks[0].completion_tokens == litellm_main.DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT + assert usage_chunks[0].total_tokens == _ADMISSION_INPUT_TOKENS + usage_chunks[0].completion_tokens + assert _prompt_token_counter_calls(token_counter) == [] + assert all(chunk.choices for chunk in chunks[:-1]) + assert {chunk.id for chunk in chunks} == {chunks[0].id} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("n", (None, 2)) +async def test_mock_acompletion_stream_usage_reports_admission_input_tokens_without_tokenizer_fallback( + n: int | None, +): + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + response = await litellm.acompletion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + n=n, + stream_options={"include_usage": True}, + litellm_metadata=_ADMISSION_METADATA, + ) + chunks = [chunk async for chunk in response] + + usage_chunks = _client_usage_chunks(chunks) + assert len(usage_chunks) == 1 + assert usage_chunks[0].prompt_tokens == _ADMISSION_INPUT_TOKENS + assert usage_chunks[0].total_tokens == _ADMISSION_INPUT_TOKENS + usage_chunks[0].completion_tokens + assert _prompt_token_counter_calls(token_counter) == [] + assert all(chunk.choices for chunk in chunks[:-1]) + assert {chunk.id for chunk in chunks} == {chunks[0].id} + + +def test_mock_completion_stream_without_include_usage_hides_usage_chunk_but_logs_admission_count(): + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + chunks = list( + litellm.completion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + metadata=_ADMISSION_METADATA, + ) + ) + + assert _client_usage_chunks(chunks) == [] + assert all(len(chunk.choices) == 1 for chunk in chunks) + assert chunks[-1]._hidden_params["usage"].prompt_tokens == _ADMISSION_INPUT_TOKENS + assert _prompt_token_counter_calls(token_counter) == [] + + +def test_mock_completion_stream_without_admission_count_falls_back_to_tokenizer(): + expected_prompt_tokens: Final = litellm.token_counter(model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES) + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + chunks = list( + litellm.completion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + stream_options={"include_usage": True}, + metadata={"user_api_key_budget_reservation": {"reserved_cost": 1.0}}, + ) + ) + + usage_chunks = _client_usage_chunks(chunks) + assert len(usage_chunks) == 1 + assert usage_chunks[0].prompt_tokens == expected_prompt_tokens + assert usage_chunks[0].total_tokens == expected_prompt_tokens + usage_chunks[0].completion_tokens + assert len(_prompt_token_counter_calls(token_counter)) >= 1 + + +@pytest.mark.asyncio +async def test_mock_acompletion_stream_without_admission_count_falls_back_to_tokenizer(): + expected_prompt_tokens: Final = litellm.token_counter(model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES) + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + response = await litellm.acompletion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + stream_options={"include_usage": True}, + ) + chunks = [chunk async for chunk in response] + + usage_chunks = _client_usage_chunks(chunks) + assert len(usage_chunks) == 1 + assert usage_chunks[0].prompt_tokens == expected_prompt_tokens + assert len(_prompt_token_counter_calls(token_counter)) >= 1 + + +def test_mock_completion_stream_and_non_stream_report_the_same_admission_usage(): + non_stream = litellm.completion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + metadata=_ADMISSION_METADATA, + ) + chunks = list( + litellm.completion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + stream_options={"include_usage": True}, + metadata=_ADMISSION_METADATA, + ) + ) + + stream_usage: Final = _client_usage_chunks(chunks)[0] + assert (non_stream.usage.prompt_tokens, non_stream.usage.completion_tokens, non_stream.usage.total_tokens) == ( + stream_usage.prompt_tokens, + stream_usage.completion_tokens, + stream_usage.total_tokens, + ) + + def test_mock_completion_stream_with_model_response(): """Test that mock_completion correctly handles stream=True with a ModelResponse as mock_response.""" from litellm import completion diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 8b186be43e5..2bdde43ff2d 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -15,6 +15,7 @@ from jsonschema import validate import litellm from litellm._internal_context import is_internal_call +from litellm.constants import DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT from litellm._logging import ( CorrelationContextFilter, JsonFormatter, @@ -6208,3 +6209,92 @@ def test_load_credentials_from_list_fills_kwargs_from_the_loaded_credential_with "api_key": "sk-from-db", } assert _credential_warnings(caplog) == [] + + +_MOCK_STREAM_ID: Final = "chatcmpl-mock-stream" +_ChunkSnapshot = tuple[str, tuple[str | None, ...], Usage | None] + + +def _snapshot(chunk: ModelResponseStream) -> _ChunkSnapshot: + return chunk.id, tuple(choice.delta.content for choice in chunk.choices), getattr(chunk, "usage", None) + + +def _mock_stream_snapshots(mock_response: object, prompt_tokens: int | None) -> list[_ChunkSnapshot]: + from litellm.utils import mock_completion_streaming_obj + + return [ + _snapshot(chunk) + for chunk in mock_completion_streaming_obj( + ModelResponseStream(id=_MOCK_STREAM_ID, model="gpt-5.4-mini"), + mock_response=mock_response, + model="gpt-5.4-mini", + prompt_tokens=prompt_tokens, + ) + ] + + +async def _async_mock_stream_snapshots(mock_response: object, prompt_tokens: int | None) -> list[_ChunkSnapshot]: + from litellm.utils import async_mock_completion_streaming_obj + + return [ + _snapshot(chunk) + async for chunk in async_mock_completion_streaming_obj( + ModelResponseStream(id=_MOCK_STREAM_ID, model="gpt-5.4-mini"), + mock_response=mock_response, + model="gpt-5.4-mini", + prompt_tokens=prompt_tokens, + ) + ] + + +_CONTENT_SNAPSHOTS: Final = [(_MOCK_STREAM_ID, (content,), None) for content in ("hel", "lo ", "wor", "ld")] + + +def _assert_trailing_usage_chunk(snapshots: list[_ChunkSnapshot], prompt_tokens: int) -> None: + assert snapshots[:-1] == _CONTENT_SNAPSHOTS + chunk_id, choices, usage = snapshots[-1] + assert chunk_id == _MOCK_STREAM_ID + assert choices == () + assert usage is not None + assert usage.prompt_tokens == prompt_tokens + assert usage.completion_tokens == DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT + assert usage.total_tokens == prompt_tokens + usage.completion_tokens + + +@pytest.mark.parametrize("prompt_tokens", (51234, 0)) +def test_mock_completion_streaming_obj_emits_usage_chunk_with_admission_prompt_tokens(prompt_tokens: int) -> None: + _assert_trailing_usage_chunk(_mock_stream_snapshots("hello world", prompt_tokens), prompt_tokens) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prompt_tokens", (51234, 0)) +async def test_async_mock_completion_streaming_obj_emits_usage_chunk_with_admission_prompt_tokens( + prompt_tokens: int, +) -> None: + _assert_trailing_usage_chunk(await _async_mock_stream_snapshots("hello world", prompt_tokens), prompt_tokens) + + +def test_mock_completion_streaming_obj_emits_no_usage_chunk_without_admission_prompt_tokens() -> None: + assert _mock_stream_snapshots("hello world", None) == _CONTENT_SNAPSHOTS + + +@pytest.mark.asyncio +async def test_async_mock_completion_streaming_obj_emits_no_usage_chunk_without_admission_prompt_tokens() -> None: + assert await _async_mock_stream_snapshots("hello world", None) == _CONTENT_SNAPSHOTS + + +def test_mock_completion_streaming_obj_passes_prebuilt_stream_chunk_through_without_usage_chunk() -> None: + prebuilt: Final = ModelResponseStream( + model="gpt-5.4-mini", choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="prebuilt"))] + ) + + assert _mock_stream_snapshots(prebuilt, 51234) == [(prebuilt.id, ("prebuilt",), None)] + + +@pytest.mark.asyncio +async def test_async_mock_completion_streaming_obj_raises_mock_exception_before_usage_chunk() -> None: + mock_exception: Final = litellm.MockException( + status_code=500, message="boom", llm_provider="openai", model="gpt-5.4-mini" + ) + with pytest.raises(litellm.MockException): + await _async_mock_stream_snapshots(mock_exception, 51234) From 985ac6b6a514d8abec939b897720326a8404eb69 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:48:46 -0700 Subject: [PATCH 105/126] fix(rate_limiter): skip non-Latin-1 x-litellm-priority header on /v1/messages (#40636) A team or key priority that is not Latin-1 encodable (for example CJK text) was attached as a response header by the dynamic rate limiter v3 post-call hook, and Starlette then raised UnicodeEncodeError while writing headers, turning a successful /v1/messages call into HTTP 500. The header is now omitted for such values while x-litellm-rate-limiter-version and the v3 rate limit headers are still attached. Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/hooks/dynamic_rate_limiter_v3.py | 12 ++++++++- .../hooks/test_dynamic_rate_limiter_v3.py | 27 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index a074f02f4e8..0339cf4dfea 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -59,6 +59,10 @@ def _get_priority_settings() -> "PriorityReservationSettings": return settings +def _is_latin1_encodable(value: object) -> bool: + return all(ord(char) < 256 for char in str(value)) + + class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): """ Saturation-aware priority-based rate limiter using v3 infrastructure. @@ -666,7 +670,13 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): if response_has_hidden_params(response): priority: Final = self._get_priority_from_user_api_key_dict(user_api_key_dict=user_api_key_dict) additional_headers: Final = ensure_response_additional_headers(response) - additional_headers["x-litellm-priority"] = priority or "default" + priority_header: Final = priority or "default" + if _is_latin1_encodable(priority_header): + additional_headers["x-litellm-priority"] = priority_header + else: + verbose_proxy_logger.debug( + "Skipping x-litellm-priority header: priority %r is not Latin-1 encodable", priority + ) additional_headers["x-litellm-rate-limiter-version"] = "v3" return response diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index 0cd6b4ede9c..527449bbc48 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -1918,3 +1918,30 @@ async def test_post_call_success_hook_leaves_raw_provider_dict_untouched(): ) assert response == {"id": "msg_123", "type": "message", "role": "assistant", "content": []} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("team_metadata", "expected_priority_header"), + [ + ({"priority": "优先"}, None), + ({"priority": "high"}, "high"), + ({}, "default"), + ], +) +async def test_post_call_success_hook_priority_header_is_always_http_encodable(team_metadata, expected_priority_header): + from starlette.responses import Response + + handler = DynamicRateLimitHandler(internal_usage_cache=DualCache()) + response = {"id": "msg_123", "type": "message", "role": "assistant", "content": [], "_hidden_params": {}} + + await handler.async_post_call_success_hook( + data={"model": "anthropic-haiku"}, + user_api_key_dict=UserAPIKeyAuth(team_id="team-1", team_metadata=team_metadata), + response=response, + ) + + additional_headers = response["_hidden_params"]["additional_headers"] + http_response = Response(headers={key: str(value) for key, value in additional_headers.items()}) + assert http_response.headers.get("x-litellm-priority") == expected_priority_header + assert http_response.headers["x-litellm-rate-limiter-version"] == "v3" From e9c388d79917cd4018a568181f31969ab223cf4e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:49:20 -0700 Subject: [PATCH 106/126] fix(caching): keep an open Redis circuit breaker quiet on the sync read and spend counter paths The sync get path was unguarded, logged with a stray format argument, and never fed the breaker. The sync batch read swallowed the breaker's refusal as an ERROR plus a service failure event per call, so DualCache dropped its in-memory hits and left batch reservations behind. record_success closed an OPEN breaker on stale in-flight successes, skipping the recovery timeout and the half-open probe. The spend counter pipeline re-raised the refusal into the cost callback, which logged an ERROR and fired the failed-tracking alert per request. --- litellm/caching/dual_cache.py | 12 ++- litellm/caching/redis_cache.py | 9 +- litellm/proxy/management_endpoints/ui_sso.py | 18 ++-- litellm/proxy/proxy_server.py | 5 +- tests/test_litellm/caching/test_dual_cache.py | 27 ++++++ .../test_litellm/caching/test_redis_cache.py | 83 +++++++++++++++++-- .../proxy/management_endpoints/test_ui_sso.py | 22 +++++ .../proxy/proxy_server/test_spend_counters.py | 48 +++++++++++ 8 files changed, 205 insertions(+), 19 deletions(-) diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index baabfad6852..be761e1258b 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -23,7 +23,7 @@ from litellm.constants import DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE from .base_cache import BaseCache from .in_memory_cache import InMemoryCache -from .redis_cache import RedisCache, log_redis_failure +from .redis_cache import RedisCache, RedisCircuitBreakerOpenError, log_redis_failure if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -206,9 +206,12 @@ class DualCache(BaseCache): redis_result: Final = self.redis_cache.batch_get_cache( key_list=sublist_keys, parent_otel_span=parent_otel_span ) - except Exception: + except Exception as e: # Do not throttle subsequent callers if the Redis read fails. self._rollback_redis_batch_key_reservations(previous_access_times) + if isinstance(e, RedisCircuitBreakerOpenError): + verbose_logger.debug("LiteLLM Cache: batch_get_cache served from memory only: %s", e) + return result raise if self.in_memory_cache is not None: @@ -325,9 +328,12 @@ class DualCache(BaseCache): redis_result: Final = await self.redis_cache.async_batch_get_cache( sublist_keys, parent_otel_span=parent_otel_span ) - except Exception: + except Exception as e: # Do not throttle subsequent callers if the Redis read fails. self._rollback_redis_batch_key_reservations(previous_access_times) + if isinstance(e, RedisCircuitBreakerOpenError): + verbose_logger.debug("LiteLLM Cache: async_batch_get_cache served from memory only: %s", e) + return result raise # Short-circuit if redis_result is None or contains only None values diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 6b93529e456..5b1a9739eb8 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -250,7 +250,7 @@ class RedisCircuitBreaker: self._set_state(self.OPEN) def record_success(self) -> None: - if not self.enabled: + if not self.enabled or self._state == self.OPEN: return if self._state == self.HALF_OPEN: verbose_logger.info("Redis circuit breaker CLOSED — Redis recovered") @@ -1337,6 +1337,7 @@ class RedisCache(BaseCache): except Exception: return ast.literal_eval(decoded) + @_redis_circuit_breaker_guard_sync def get_cache(self, key, parent_otel_span: Span | None = None, **kwargs): try: key = self.check_and_fix_namespace(key=key) @@ -1356,8 +1357,8 @@ class RedisCache(BaseCache): print_verbose(f"Got Redis Cache: key: {key}, cached_response {cached_response}") return self._get_cache_logic(cached_response=cached_response) except Exception as e: - # NON blocking - notify users Redis is throwing an exception - verbose_logger.error("litellm.caching.caching: get() - Got exception from REDIS: ", e) + verbose_logger.error("litellm.caching.caching: get() - Got exception from REDIS: %s", e) + _record_swallowed_redis_failure(self._circuit_breaker, e) def _run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]: """ @@ -1394,9 +1395,9 @@ class RedisCache(BaseCache): key_value_dict = {} _key_list: Final = [key for key in key_list if key is not None] start_time: Final = time.time() + swallowed_before: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache") try: - swallowed_before: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache") _keys: Final = [self.check_and_fix_namespace(key=cache_key or "") for cache_key in _key_list] results: Final = self._run_redis_mget_operation(keys=_keys) _exit_circuit_breaker(self._circuit_breaker, swallowed_before) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index c60888e298f..1ba90725eff 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -47,6 +47,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.caching.dual_cache import DualCache +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.constants import ( CLI_SSO_CLAIM_MAP, CLI_SSO_CLAIM_MAX_SCALAR_LENGTH, @@ -336,6 +337,16 @@ def _check_cli_sso_start_rate_limit( ) +def _read_cli_sso_flow(cache: DualCache, cache_key: str) -> object: + redis_cache: Final = cache.redis_cache + if redis_cache is None: + return cache.get_cache(key=cache_key) + try: + return redis_cache.get_cache(key=cache_key) + except RedisCircuitBreakerOpenError: + return None + + def _get_cli_sso_flow_or_raise(login_id: str | None, cache: DualCache) -> dict: if isinstance(login_id, str) and login_id.startswith("sk-"): raise HTTPException( @@ -348,12 +359,7 @@ def _get_cli_sso_flow_or_raise(login_id: str | None, cache: DualCache) -> dict: if not _is_valid_cli_sso_login_id(login_id): raise HTTPException(status_code=400, detail="Invalid CLI login session id") - cache_key: Final = _get_cli_sso_flow_cache_key(cast(str, login_id)) - redis_cache: Final = cache.redis_cache - if redis_cache is not None: - flow = redis_cache.get_cache(key=cache_key) - else: - flow = cache.get_cache(key=cache_key) + flow = _read_cli_sso_flow(cache, _get_cli_sso_flow_cache_key(cast(str, login_id))) if isinstance(flow, str): try: flow = _as_object(json.loads(flow)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0f24acb8bb4..94e74b20297 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -251,6 +251,7 @@ import litellm._redis from litellm import Router from litellm._logging import _redact_string, verbose_proxy_logger, verbose_router_logger from litellm.caching.caching import DualCache, RedisCache +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.caching.redis_cluster_cache import RedisClusterCache from litellm.constants import ( _REALTIME_BODY_CACHE_SIZE, @@ -3412,8 +3413,10 @@ async def _apply_spend_counter_increments(pending: Sequence[_PendingSpendIncreme ] try: results: Final = await redis_cache.async_increment_pipeline(increment_list=increment_list) - except Exception: + except Exception as e: await asyncio.gather(*(_invalidate_spend_counter(counter_key=item.counter_key) for item in pending)) + if isinstance(e, RedisCircuitBreakerOpenError): + return raise for item, current_value in zip(pending, results or ()): spend_counter_cache.in_memory_cache.set_cache(key=item.counter_key, value=current_value) diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index eae8bbfdaff..4c9068722b8 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -677,3 +677,30 @@ async def test_a_real_redis_failure_still_logs_an_error(caplog): errors = [record for record in caplog.records if record.levelno == logging.ERROR] assert [record.getMessage() for record in errors] == ["LiteLLM Cache: exception in async_get_cache: redis is down"] assert errors[0].exc_info is not None + + +def _dual_cache_with_open_breaker_and_a_memory_hit() -> DualCache: + in_memory = InMemoryCache() + in_memory.set_cache("k1", "v1") + return DualCache(in_memory_cache=in_memory, redis_cache=_OpenBreakerRedis(), default_redis_batch_cache_expiry=10) # pyright: ignore[reportArgumentType] # duck-typed Redis double + + +def test_open_breaker_keeps_sync_batch_read_memory_hits_and_releases_reservations(): + """A refused Redis batch read must still answer with the in-memory hits and hold no reservation. + + The refusal was logged and turned into a bare None, so a caller lost its in-memory hits + for as long as the breaker stayed open, and the reserved keys stayed throttled until + the batch expiry passed even though nothing was ever read for them. + """ + cache = _dual_cache_with_open_breaker_and_a_memory_hit() + + assert list(cache.batch_get_cache(["k1", "k2"])) == ["v1", None] + assert "k2" not in cache.last_redis_batch_access_time + + +@pytest.mark.asyncio +async def test_open_breaker_keeps_async_batch_read_memory_hits_and_releases_reservations(): + cache = _dual_cache_with_open_breaker_and_a_memory_hit() + + assert list(await cache.async_batch_get_cache(["k1", "k2"])) == ["v1", None] + assert "k2" not in cache.last_redis_batch_access_time diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index bcaa58c9c40..c6f0caa1dca 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -1,11 +1,12 @@ import asyncio +import time from collections.abc import Iterator from unittest.mock import AsyncMock, MagicMock, patch import pytest from litellm._service_logger import ServiceLogging -from litellm.caching.redis_cache import RedisCache +from litellm.caching.redis_cache import RedisCache, RedisCircuitBreakerOpenError @pytest.fixture @@ -515,14 +516,46 @@ async def test_circuit_breaker_opens_when_method_swallows_redis_failure(call_met await call_method(cache) -def test_circuit_breaker_open_keeps_sync_batch_get_cache_as_a_miss(sync_batch_redis_cache): - """An open breaker must preserve the sync batch read's dictionary fallback.""" +def test_circuit_breaker_open_makes_sync_batch_get_cache_fast_fail(sync_batch_redis_cache, caplog): + """Once the breaker is open the sync batch read refuses with the typed error instead of a miss. + + Swallowing the refusal into `{}` made every sync batch read on an open breaker emit an ERROR + log and a service failure event per call, and the DualCache caller could not tell the + refusal from a dead Redis, so it dropped its in-memory hits too. + """ from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD for _ in range(REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD): assert sync_batch_redis_cache.batch_get_cache(key_list=["lit6729"]) == {} - assert sync_batch_redis_cache.batch_get_cache(key_list=["lit6729"]) == {} + caplog.clear() + with caplog.at_level("INFO"): + with pytest.raises(RedisCircuitBreakerOpenError): + sync_batch_redis_cache.batch_get_cache(key_list=["lit6729"]) + sync_batch_redis_cache.redis_client.mget.assert_called() + assert caplog.records == [] + + +def test_sync_get_cache_failure_feeds_the_breaker_and_logs_a_well_formed_record(sync_batch_redis_cache, caplog): + """The sync get path swallowed its Redis error without recording it, and its log call was malformed. + + `verbose_logger.error("...: ", e)` passes the exception as a format argument to a message + with no placeholder, so the record carried no error text. Nothing fed the breaker either, + so a dead Redis read through this path never opened it. + """ + from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD + + sync_batch_redis_cache.redis_client.get.side_effect = OSError("redis unavailable") + + with caplog.at_level("ERROR"): + for _ in range(REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD): + assert sync_batch_redis_cache.get_cache("lit7468") is None + + assert all("redis unavailable" in record.getMessage() for record in caplog.records) + assert len(caplog.records) == REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD + assert sync_batch_redis_cache._circuit_breaker.is_open() is True + with pytest.raises(RedisCircuitBreakerOpenError): + sync_batch_redis_cache.get_cache("lit7468") def test_batch_get_counts_raises_where_batch_get_cache_reports_a_miss(sync_batch_redis_cache): @@ -661,7 +694,8 @@ def test_sync_batch_get_cache_survives_a_service_callback_that_raises( with ThreadPoolExecutor(max_workers=1) as pool: assert pool.submit(cache.batch_get_cache, key_list=["lit6729"]).result() == {} - assert cache.batch_get_cache(key_list=["lit6729"]) == {} + with pytest.raises(RedisCircuitBreakerOpenError): + cache.batch_get_cache(key_list=["lit6729"]) def test_call_stack_info_skips_breaker_guard_frames(): @@ -1010,6 +1044,8 @@ async def test_breaker_metrics_track_state_and_failure_class(): assert sample("litellm_redis_circuit_breaker_state", {"state": "open"}) == open_gauge_before + 1 assert sample("litellm_redis_circuit_breaker_state", {"state": "closed"}) == closed_gauge_before + breaker._opened_at = time.time() - 9999 + assert breaker.is_open() is False breaker.record_success() assert sample("litellm_redis_circuit_breaker_state", {"state": "open"}) == open_gauge_before assert sample("litellm_redis_circuit_breaker_state", {"state": "closed"}) == closed_gauge_before + 1 @@ -1030,3 +1066,40 @@ def test_sync_guard_counts_a_timeout_as_a_timeout(): _run_under_circuit_breaker_sync(breaker, "op", timing_out_call) assert breaker.is_open() is False + + +def test_success_admitted_before_the_breaker_opened_cannot_close_it(): + """A stale in-flight success must not close a breaker that opened while it ran. + + Calls admitted while the breaker was still closed finish after later failures opened it. + Recording their success unconditionally closed the breaker again, skipping the recovery + timeout and the single half-open probe, so the breaker flapped between open and closed + on every straggler while Redis was still down. + """ + from litellm.caching.redis_cache import RedisCircuitBreaker + + breaker = RedisCircuitBreaker(failure_threshold=3, recovery_timeout=60) + for _ in range(3): + breaker.record_failure() + assert breaker._state == breaker.OPEN + + breaker.record_success() + + assert breaker._state == breaker.OPEN + assert breaker.is_open() is True + + +def test_recovery_probe_still_closes_the_breaker(): + from litellm.caching.redis_cache import RedisCircuitBreaker + + breaker = RedisCircuitBreaker(failure_threshold=3, recovery_timeout=60) + for _ in range(3): + breaker.record_failure() + breaker._opened_at = time.time() - 9999 + assert breaker.is_open() is False + assert breaker._state == breaker.HALF_OPEN + + breaker.record_success() + + assert breaker._state == breaker.CLOSED + assert breaker.is_open() is False diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 8d8bc15f9be..2050e65d2a1 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -2616,6 +2616,28 @@ class TestCLIKeyRegenerationFlow: ) cache.set_cache.assert_not_called() + def test_cli_sso_flow_lookup_treats_an_open_redis_breaker_as_a_miss(self): + """A Redis read refused by the open circuit breaker is a missing session, not a server error. + + The direct Redis read is what keeps the flow authoritative across workers, so the + refusal must not fall back to a possibly stale in-memory copy either. + """ + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError + from litellm.proxy.management_endpoints.ui_sso import _get_cli_sso_flow_or_raise + + redis_cache = MagicMock() + redis_cache.get_cache.side_effect = RedisCircuitBreakerOpenError("Redis circuit breaker is open") + cache = MagicMock() + cache.redis_cache = redis_cache + cache.get_cache.return_value = {"poll_secret_hash": "stale", "sso_complete": False} + + with pytest.raises(HTTPException) as exc_info: + _get_cli_sso_flow_or_raise(login_id="cli-breaker_open_1234567890", cache=cache) + + assert exc_info.value.status_code == 400 + assert "not found or expired" in exc_info.value.detail + cache.get_cache.assert_not_called() + def test_cli_sso_flow_with_enum_survives_redis_round_trip(self): """ RedisCache stores values via str(value) and reads them back through diff --git a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py index c343652efd9..2f47736a398 100644 --- a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py +++ b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py @@ -1146,6 +1146,54 @@ async def test_prepare_window_spend_counter_increment_missing_window_start_inval assert fake_cache.redis_cache.async_increment.called is False +# --------------------------------------------------------------------------- +# _apply_spend_counter_increments +# --------------------------------------------------------------------------- + + +def _two_pending_increments() -> tuple[ps._PendingSpendIncrement, ...]: + return ( + ps._PendingSpendIncrement(counter_key="spend:key:k", increment=1.5), + ps._PendingSpendIncrement(counter_key="spend:team:t", increment=1.5), + ) + + +@pytest.mark.asyncio +async def test_apply_spend_counter_increments_open_breaker_invalidates_and_returns(monkeypatch): + """An open Redis circuit breaker is a known, already-logged state, not a per-request tracking failure. + + Re-raising the refusal sent every request through the cost callback's error path, which + logged an ERROR and fired the failed-tracking alert once per request for the whole outage. + """ + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError + + fake_cache = _make_spend_counter_cache() + fake_cache.redis_cache.async_increment_pipeline = AsyncMock( + side_effect=RedisCircuitBreakerOpenError("Redis circuit breaker is open") + ) + monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) + + await ps._apply_spend_counter_increments(_two_pending_increments()) + + deleted_keys = sorted(call.kwargs["key"] for call in fake_cache.in_memory_cache.delete_cache.call_args_list) + assert deleted_keys == ["spend:key:k", "spend:team:t"] + fake_cache.in_memory_cache.set_cache.assert_not_called() + + +@pytest.mark.asyncio +async def test_apply_spend_counter_increments_other_redis_error_invalidates_and_raises(monkeypatch): + fake_cache = _make_spend_counter_cache() + fake_cache.redis_cache.async_increment_pipeline = AsyncMock(side_effect=ConnectionError("redis down")) + monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) + + with pytest.raises(ConnectionError, match="redis down"): + await ps._apply_spend_counter_increments(_two_pending_increments()) + + deleted_keys = sorted(call.kwargs["key"] for call in fake_cache.in_memory_cache.delete_cache.call_args_list) + assert deleted_keys == ["spend:key:k", "spend:team:t"] + fake_cache.in_memory_cache.set_cache.assert_not_called() + + # --------------------------------------------------------------------------- # _ensure_spend_counter_initialized # --------------------------------------------------------------------------- From 5ea2f9698295a0e53735a9f8a001262de5a7dee6 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 17:59:22 -0700 Subject: [PATCH 107/126] feat(ui): link the Organization cell on the Virtual Keys page too Same treatment as User, Team and Created By in the previous commit: the Organization column rendered the alias as dead text, so it now goes through IdentityCell with an orgDetailHref. Claude-Session: https://claude.ai/code/session_01NfwfQhamRNnSqgXMUjf3h4 --- .../VirtualKeysPage/VirtualKeysTable.test.tsx | 19 +++++++++++++++++-- .../VirtualKeysPage/keyTableColumns.tsx | 12 ++++++------ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index 57786d71c66..c303dba697b 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -476,12 +476,14 @@ it("should display 'Default Proxy Admin' for user_id when value is 'default_user describe("entity links out of the key rows", () => { const keyRow = async () => (await screen.findByText("Test Key Alias")).closest("tr") as HTMLElement; - const enableCreatedByColumn = async (user: ReturnType) => { + const enableColumn = async (user: ReturnType, title: string) => { await user.click(screen.getByRole("button", { name: "Columns" })); - await user.click(await screen.findByText("Created By")); + await user.click(await screen.findByText(title)); await user.keyboard("{Escape}"); }; + const enableCreatedByColumn = (user: ReturnType) => enableColumn(user, "Created By"); + it("points the User and Team cells at their detail pages", async () => { renderWithProviders(); @@ -493,6 +495,19 @@ describe("entity links out of the key rows", () => { expect(within(row).getByRole("link", { name: "Test Team" })).toHaveAttribute("href", "/ui/teams?team=team-1"); }); + it("points the Organization cell at the org's detail page", async () => { + mockUseKeys.mockReturnValue(keysResult([{ ...mockKey, org_id: "org-1" }])); + const user = userEvent.setup(); + renderWithProviders(); + await enableColumn(user, "Organization"); + + const row = await keyRow(); + expect(within(row).getByRole("link", { name: "Test Organization" })).toHaveAttribute( + "href", + "/ui/organizations?org=org-1", + ); + }); + it("points the Created By cell at the creator's detail page", async () => { mockUseKeys.mockReturnValue( keysResult([ diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx index c87bc90fb47..9dd4b1c2d59 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx @@ -16,7 +16,7 @@ import { StatusBadge, type StatusTone, } from "@/components/shared/table_cells"; -import { teamDetailHref, userDetailHref } from "@/utils/entityLinks"; +import { orgDetailHref, teamDetailHref, userDetailHref } from "@/utils/entityLinks"; import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; @@ -218,12 +218,12 @@ export const getKeyTableColumns = ({ const orgId = info.getValue() as string | null; if (!orgId) return "-"; const org = organizations.find((o) => o.organization_id === orgId); - const displayValue = org?.organization_alias || orgId; - const width = info.cell.column.getSize(); return ( - - {displayValue} - + ); }, }, From d1a1cda144391eefecd21ecc6d8d096af04f8782 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 10 Sep 2026 18:07:12 -0700 Subject: [PATCH 108/126] test: respect optional logging payload fields --- tests/local_testing/test_custom_callback_input.py | 4 ++-- tests/logging_callback_tests/test_datadog.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/local_testing/test_custom_callback_input.py b/tests/local_testing/test_custom_callback_input.py index f0f24a6e6b2..834570091bd 100644 --- a/tests/local_testing/test_custom_callback_input.py +++ b/tests/local_testing/test_custom_callback_input.py @@ -1076,7 +1076,7 @@ def test_standard_logging_payload(model, turn_off_message_logging): ) ) - keys_list = list(StandardLoggingPayload.__annotations__.keys()) + keys_list = list(StandardLoggingPayload.__required_keys__) for k in keys_list: assert ( @@ -1190,7 +1190,7 @@ def test_standard_logging_payload_audio(turn_off_message_logging, stream): ) ) - keys_list = list(StandardLoggingPayload.__annotations__.keys()) + keys_list = list(StandardLoggingPayload.__required_keys__) for k in keys_list: assert ( diff --git a/tests/logging_callback_tests/test_datadog.py b/tests/logging_callback_tests/test_datadog.py index 83a652e8884..7ac9ac0b5ad 100644 --- a/tests/logging_callback_tests/test_datadog.py +++ b/tests/logging_callback_tests/test_datadog.py @@ -270,7 +270,7 @@ async def test_datadog_logging_http_request(): message = json.loads(body[0]["message"]) print("logged message", json.dumps(message, indent=4)) - expected_message_fields = StandardLoggingPayload.__annotations__.keys() + expected_message_fields = StandardLoggingPayload.__required_keys__ for field in expected_message_fields: assert field in message, f"Field '{field}' is missing from the message" From 4fbe2276a1ece2f58d047c60660cfe630fdbbede Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:15:24 -0700 Subject: [PATCH 109/126] fix(logging): finish response metadata before the sync logging thread reads it (#39869) * fix(logging): finish response metadata before the sync logging thread reads it The async and sync client wrappers handed the response to the threaded success handler before computing its cost, call id, and api_base, so that thread inserted into the same metadata dict the request coroutine was still iterating and a finished chat completion turned into a 500 (dictionary changed size during iteration). Metadata is now finalized first, and the merge and header copies snapshot their dicts before iterating. * fix(logging): snapshot metadata with a dict copy and drop redundant comment Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): copy metadata via dict.copy and dedupe Final import Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 39 +++--- litellm/utils.py | 112 +++++++++++------- .../test_litellm_logging.py | 59 +++++++++ tests/test_litellm/test_utils.py | 46 +++++++ 4 files changed, 193 insertions(+), 63 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 893bfeefff8..6ee68ab21c5 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -295,6 +295,9 @@ def _get_provider_request_id(original_exception: Exception) -> str | None: # Cache custom pricing keys as frozenset for O(1) lookups instead of looping through 49 keys _CUSTOM_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields.keys()) _MODEL_INFO_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = _CUSTOM_PRICING_KEYS | DEPLOYMENT_SCOPED_PRICING_FIELDS +_UNSERIALIZABLE_METADATA_KEYS: Final[frozenset[str]] = frozenset( + ("user_api_key_auth", "user_api_key_budget_reservation") +) sentry_sdk_instance = None capture_exception = None @@ -5386,23 +5389,23 @@ class StandardLoggingPayloadSetup: Returns: dict: Merged metadata with user API key fields taking precedence """ - merged_metadata: Final[dict] = {} - - # Start with metadata (user API key fields) - but skip non-serializable objects - if litellm_params.get("metadata") and isinstance(litellm_params.get("metadata"), dict): - for key, value in litellm_params["metadata"].items(): - # Skip non-serializable objects like UserAPIKeyAuth - if key in {"user_api_key_auth", "user_api_key_budget_reservation"}: - continue - merged_metadata[key] = value - - # Then merge litellm_metadata (model-related fields) - this will NOT overwrite existing keys - if litellm_params.get("litellm_metadata") and isinstance(litellm_params.get("litellm_metadata"), dict): - for key, value in litellm_params["litellm_metadata"].items(): - if key not in merged_metadata: # Don't overwrite existing keys from metadata - merged_metadata[key] = value - - return merged_metadata + metadata: Final = litellm_params.get("metadata") + litellm_metadata: Final = litellm_params.get("litellm_metadata") + user_metadata: Final = MappingProxyType( + { + key: value + for key, value in (metadata.copy().items() if isinstance(metadata, dict) else ()) + if key not in _UNSERIALIZABLE_METADATA_KEYS + } + ) + model_metadata: Final = MappingProxyType( + { + key: value + for key, value in (litellm_metadata.copy().items() if isinstance(litellm_metadata, dict) else ()) + if key not in user_metadata + } + ) + return {**user_metadata, **model_metadata} # mutable-ok: function contract returns a plain dict @staticmethod def get_standard_logging_metadata( @@ -5660,7 +5663,7 @@ class StandardLoggingPayloadSetup: additional_logging_headers[key] = additiona_headers[_key] # Preserve all remaining headers verbatim (e.g. llm_provider-x-request-id) - for k, v in additiona_headers.items(): + for k, v in additiona_headers.copy().items(): if k.lower() not in typed_keys: additional_logging_headers[k] = v diff --git a/litellm/utils.py b/litellm/utils.py index b93938a480e..a765e1b1246 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1196,6 +1196,47 @@ def function_setup( raise e +def _dispatch_success_logging( + logging_obj: LiteLLMLoggingObject, + result: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + is_completion_with_fallbacks: bool, + is_litellm_internal_call: bool, +) -> None: + if not is_litellm_internal_call: + if getattr(logging_obj, "_defer_async_logging", False): + + def _enqueue_deferred_logging() -> None: + asyncio.create_task( + _client_async_logging_helper( + logging_obj=logging_obj, + result=result, + start_time=start_time, + end_time=end_time, + is_completion_with_fallbacks=is_completion_with_fallbacks, + ) + ) + + logging_obj._enqueue_deferred_logging = _enqueue_deferred_logging + else: + asyncio.create_task( + _client_async_logging_helper( + logging_obj=logging_obj, + result=result, + start_time=start_time, + end_time=end_time, + is_completion_with_fallbacks=is_completion_with_fallbacks, + ) + ) + + logging_obj.handle_sync_success_callbacks_for_async_calls( + result=result, + start_time=start_time, + end_time=end_time, + ) + + async def _client_async_logging_helper( logging_obj: LiteLLMLoggingObject, result, @@ -1663,6 +1704,16 @@ def client(original_function): kwargs=kwargs, ) + _update_response_metadata: Final = getattr(sys.modules[__name__], "update_response_metadata") + _update_response_metadata( + result=result, + logging_obj=logging_obj, + model=model, + kwargs=kwargs, + start_time=start_time, + end_time=end_time, + ) + # LOG SUCCESS - handle streaming success logging in the _next_ object, remove `handle_success` once it's deprecated verbose_logger.info("Wrapper: Completed Call, calling success_handler") # Copy the current context to propagate it to the background thread @@ -1677,15 +1728,6 @@ def client(original_function): end_time, ) # RETURN RESULT - update_response_metadata = getattr(sys.modules[__name__], "update_response_metadata") - update_response_metadata( - result=result, - logging_obj=logging_obj, - model=model, - kwargs=kwargs, - start_time=start_time, - end_time=end_time, - ) return result except Exception as e: call_type = original_function.__name__ @@ -1945,48 +1987,20 @@ def client(original_function): args=args, ) - # LOG SUCCESS - handle streaming success logging in the _next_ object - # Internal sub-calls (e.g. emulated file-search steps) share the - # parent's logging obj; skip async logging here so only the outer call bills once. - # NOTE: streaming requests return early (before this point) via - # CustomStreamWrapper, so this block is non-streaming only. - if not _is_litellm_internal_call: - if getattr(logging_obj, "_defer_async_logging", False): - - def _enqueue_deferred_logging() -> None: - asyncio.create_task( - _client_async_logging_helper( - logging_obj=logging_obj, - result=result, - start_time=start_time, - end_time=end_time, - is_completion_with_fallbacks=is_completion_with_fallbacks, - ) - ) - - logging_obj._enqueue_deferred_logging = _enqueue_deferred_logging - else: - asyncio.create_task( - _client_async_logging_helper( - logging_obj=logging_obj, - result=result, - start_time=start_time, - end_time=end_time, - is_completion_with_fallbacks=is_completion_with_fallbacks, - ) - ) - - logging_obj.handle_sync_success_callbacks_for_async_calls( - result=result, - start_time=start_time, - end_time=end_time, - ) # REBUILD EMBEDDING CACHING if ( isinstance(result, EmbeddingResponse) and _caching_handler_response is not None and _caching_handler_response.final_embedding_cached_response is not None ): + _dispatch_success_logging( + logging_obj=logging_obj, + result=result, + start_time=start_time, + end_time=end_time, + is_completion_with_fallbacks=is_completion_with_fallbacks, + is_litellm_internal_call=_is_litellm_internal_call, + ) return _llm_caching_handler._combine_cached_embedding_response_with_api_result( _caching_handler_response=_caching_handler_response, embedding_response=result, @@ -2002,6 +2016,14 @@ def client(original_function): start_time=start_time, end_time=end_time, ) + _dispatch_success_logging( + logging_obj=logging_obj, + result=result, + start_time=start_time, + end_time=end_time, + is_completion_with_fallbacks=is_completion_with_fallbacks, + is_litellm_internal_call=_is_litellm_internal_call, + ) return result except Exception as e: diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index aeaa8f18c26..2f6339dcdbb 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -3,6 +3,7 @@ import contextlib import datetime import os import sys +from collections.abc import Callable from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -6945,3 +6946,61 @@ def test_classifier_audit_is_not_added_to_other_calls(logging_obj, call_type, or logging_obj.model_call_details["litellm_params"] = {"metadata": {"internal_call_origin": origin}} logging_obj.pre_call(input=[], api_key=None, additional_args={"complete_input_dict": {"input": "embedding"}}) assert logging_obj.classifier_input is None + + +def _run_while_a_thread_grows(target: dict, read: Callable[[], None], reads: int) -> None: + import itertools + import threading + + stop: Final = threading.Event() + + def grow() -> None: + for counter in itertools.count(): + if stop.is_set(): + return + key: Final = f"late_{counter % 64}" + if key in target: + del target[key] + else: + target[key] = counter + + writer: Final = threading.Thread(target=grow, daemon=True) + previous_interval: Final = sys.getswitchinterval() + sys.setswitchinterval(1e-6) + writer.start() + try: + for _ in range(reads): + read() + finally: + stop.set() + writer.join(timeout=5) + sys.setswitchinterval(previous_interval) + + +def test_merge_litellm_metadata_survives_a_thread_growing_metadata_mid_merge(): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + metadata: Final = {f"key_{i}": i for i in range(2000)} + litellm_params: Final = {"metadata": metadata, "litellm_metadata": {"model_group": "gpt"}} + + def read() -> None: + merged: Final = StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params) + assert merged["key_1999"] == 1999 + assert merged["model_group"] == "gpt" + + _run_while_a_thread_grows(metadata, read, reads=300) + + +def test_get_additional_headers_survives_a_thread_growing_headers_mid_copy(): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + headers: Final = {f"llm_provider-x-custom-{i}": str(i) for i in range(2000)} + headers["x-ratelimit-remaining-requests"] = "7" + + def read() -> None: + copied: Final = StandardLoggingPayloadSetup.get_additional_headers(headers) + assert copied is not None + assert copied["x_ratelimit_remaining_requests"] == 7 + assert copied["llm_provider-x-custom-1999"] == "1999" + + _run_while_a_thread_grows(headers, read, reads=300) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 2bdde43ff2d..7753cb7d770 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1,9 +1,12 @@ import asyncio +import contextlib import json import logging import os +import queue import threading from datetime import datetime, timedelta, timezone +from collections.abc import Iterator from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -6298,3 +6301,46 @@ async def test_async_mock_completion_streaming_obj_raises_mock_exception_before_ ) with pytest.raises(litellm.MockException): await _async_mock_stream_snapshots(mock_exception, 51234) + + + +@contextlib.contextmanager +def _recording_hidden_params_at_submit(submit_target: str) -> "Iterator[queue.SimpleQueue[dict[str, object]]]": + seen: Final = queue.SimpleQueue() + + def record_submit(_fn, *args, **_kwargs): + response: Final = next(arg for arg in args if isinstance(arg, litellm.ModelResponse)) + seen.put(dict(response._hidden_params)) + return MagicMock() + + with patch(submit_target, side_effect=record_submit): + yield seen + + +@pytest.mark.asyncio +async def test_acompletion_finishes_response_metadata_before_handing_the_response_to_the_logging_thread(monkeypatch): + monkeypatch.setattr(litellm, "success_callback", [lambda kwargs, response, start_time, end_time: None]) + with _recording_hidden_params_at_submit("litellm.litellm_core_utils.litellm_logging.executor.submit") as seen: + await litellm.acompletion( + model="gpt-5.5", + messages=[{"role": "user", "content": "hi"}], + mock_response="Hello there!", + num_retries=0, + ) + snapshot: Final = seen.get_nowait() + assert snapshot["litellm_call_id"] + assert snapshot["response_cost"] is not None + assert snapshot["api_base"] + + +def test_completion_finishes_response_metadata_before_handing_the_response_to_the_logging_thread(): + with _recording_hidden_params_at_submit("litellm.utils.executor.submit") as seen: + litellm.completion( + model="gpt-5.5", + messages=[{"role": "user", "content": "hi"}], + mock_response="Hello there!", + ) + snapshot: Final = seen.get_nowait() + assert snapshot["litellm_call_id"] + assert snapshot["response_cost"] is not None + assert snapshot["api_base"] From 7169ddaef6ad4847f630f2c2bcefa2799db19c92 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:16:59 -0700 Subject: [PATCH 110/126] fix(router): treat a breaker-refused Redis read as a miss in the health state cache The sync Redis read now raises while the circuit breaker is open, and the health state merge caught that as a generic error, skipping the local write and logging an error on every background health check cycle. Read the shared snapshot through a helper that treats the refused read as a miss so the merge falls back to the pod-local copy the way a swallowed connection error already did --- litellm/router_utils/health_state_cache.py | 20 ++++++++++---- .../router_utils/test_health_state_cache.py | 27 +++++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/litellm/router_utils/health_state_cache.py b/litellm/router_utils/health_state_cache.py index 22d816e13e9..c8ca7105392 100644 --- a/litellm/router_utils/health_state_cache.py +++ b/litellm/router_utils/health_state_cache.py @@ -12,6 +12,7 @@ from typing_extensions import TypedDict from litellm import verbose_logger from litellm.caching.caching import DualCache +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -27,6 +28,16 @@ class DeploymentHealthStateValue(TypedDict): reason: str +def _read_shared_health_snapshot(cache: DualCache, key: str) -> object: + redis_cache: Final = cache.redis_cache + if redis_cache is None: + return None + try: + return redis_cache.get_cache(key) + except RedisCircuitBreakerOpenError: + return None + + class DeploymentHealthCache: """ Cache for deployment health states produced by background health checks. @@ -50,13 +61,12 @@ class DeploymentHealthCache: coexist on the one shared entry without erasing each other's results. The snapshot is read from Redis when available, since a pod-local read would only ever see this writer's own previous merge. When the Redis - read comes back empty (a miss, or a swallowed connection error), the - pod-local copy of the last merge is used so peers are not erased. + read comes back empty (a miss, a swallowed connection error, or a read + refused by the open circuit breaker), the pod-local copy of the last + merge is used so peers are not erased. """ try: - redis_raw: Final = ( - self.cache.redis_cache.get_cache(self.CACHE_KEY) if self.cache.redis_cache is not None else None - ) + redis_raw: Final = _read_shared_health_snapshot(self.cache, self.CACHE_KEY) raw: Final = redis_raw if isinstance(redis_raw, dict) else self.cache.get_cache(key=self.CACHE_KEY) existing: Final = raw if isinstance(raw, dict) else {} expiry_seconds: Final = self.staleness_threshold * 1.5 diff --git a/tests/test_litellm/router_utils/test_health_state_cache.py b/tests/test_litellm/router_utils/test_health_state_cache.py index ffd031f9b7d..aa976bb1002 100644 --- a/tests/test_litellm/router_utils/test_health_state_cache.py +++ b/tests/test_litellm/router_utils/test_health_state_cache.py @@ -7,6 +7,7 @@ import time import pytest from litellm.caching.caching import DualCache +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.router_utils.health_state_cache import DeploymentHealthCache @@ -145,8 +146,11 @@ class _SharedRedisFake: def __init__(self): self.store = {} self.fail_get = False + self.breaker_open = False def get_cache(self, key, parent_otel_span=None, **kwargs): + if self.breaker_open: + raise RedisCircuitBreakerOpenError("Redis circuit breaker is open - skipping get_cache") if self.fail_get: return None # RedisCache.get_cache swallows connection errors and returns None return self.store.get(key) @@ -192,3 +196,26 @@ def test_failed_redis_read_falls_back_to_local_copy(): {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}} ) assert set(redis_fake.store[DeploymentHealthCache.CACHE_KEY]) == {"prod-bad", "internal-bad"} + + +def test_open_circuit_breaker_read_still_merges_into_local_copy(caplog): + """A read refused by the open breaker is a miss, so the merge and local write still happen quietly.""" + redis_fake = _SharedRedisFake() + pod_a = DeploymentHealthCache(cache=DualCache(redis_cache=redis_fake), staleness_threshold=60.0) + pod_b = DeploymentHealthCache(cache=DualCache(redis_cache=redis_fake), staleness_threshold=60.0) + pod_a.set_deployment_health_states( + {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}} + ) + pod_b.set_deployment_health_states( + {"internal-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "timeout"}} + ) + pod_a.set_deployment_health_states( + {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}} + ) + redis_fake.breaker_open = True + with caplog.at_level("ERROR"): + pod_a.set_deployment_health_states( + {"prod-new-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}} + ) + assert caplog.records == [] + assert pod_a.get_unhealthy_deployment_ids() == {"prod-bad", "internal-bad", "prod-new-bad"} From dcdd884352b1f39f51dd0719bba4eca0046859f8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:36:19 -0700 Subject: [PATCH 111/126] fix(caching): let only the recovery probe close a half-open Redis breaker A call admitted before the breaker opened could finish while the breaker was HALF_OPEN and close it before the designated probe reported, so Redis traffic resumed on a stale answer. The admission now records whether the call is the probe and only the probe's success closes a half-open breaker. The cron job lock manager also logged an error every cycle the open breaker refused its Redis call, one line per job per pod. That refusal is now a debug line like every other guarded call, while real Redis errors still log at error --- litellm/caching/redis_cache.py | 43 ++++++++++------ .../db_transaction_queue/pod_lock_manager.py | 7 +-- .../test_litellm/caching/test_redis_cache.py | 49 +++++++++++++++++++ .../test_pod_lock_manager.py | 18 +++++++ 4 files changed, 100 insertions(+), 17 deletions(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 5b1a9739eb8..812b435ae5f 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -18,6 +18,7 @@ import logging import time from collections.abc import Awaitable, Callable, Sequence from contextvars import ContextVar +from dataclasses import dataclass from datetime import timedelta from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast @@ -198,6 +199,9 @@ class RedisCircuitBreaker: self._state = self.CLOSED _breaker_metrics().record_state_change(None, self._state) + def is_half_open(self) -> bool: + return self._state == self.HALF_OPEN + def is_open(self) -> bool: """Returns True if Redis calls should be skipped.""" if not self.enabled: @@ -405,21 +409,32 @@ def log_redis_failure( logger.log(level, "%s: %s", message, exc, exc_info=exc if with_traceback else None) -def _enter_circuit_breaker(breaker: RedisCircuitBreaker, name: str) -> int: - """Reject the call if the breaker is open, else return the swallowed-failure count to compare against.""" +@dataclass(frozen=True, slots=True) +class _BreakerAdmission: + swallowed_before: int + is_probe: bool + + +def _enter_circuit_breaker(breaker: RedisCircuitBreaker, name: str) -> _BreakerAdmission: + """Reject the call if the breaker is open, else record what its success may later prove.""" if breaker.is_open(): raise RedisCircuitBreakerOpenError(f"Redis circuit breaker is open — skipping {name}") - return _swallowed_redis_failures.get() + return _BreakerAdmission(swallowed_before=_swallowed_redis_failures.get(), is_probe=breaker.is_half_open()) -def _exit_circuit_breaker(breaker: RedisCircuitBreaker, swallowed_before: int) -> None: - """Record success only when nothing failed while the call ran. +def _exit_circuit_breaker(breaker: RedisCircuitBreaker, admission: _BreakerAdmission) -> None: + """Record success only when nothing failed while the call ran and the call may vouch for Redis. Several Redis methods catch their own connection errors and return a default, so a - method that returned is not on its own proof of a healthy Redis. + method that returned is not on its own proof of a healthy Redis. While the breaker is + half open only the designated recovery probe may close it: a call admitted before the + breaker opened that finishes late says nothing about whether Redis recovered. """ - if _swallowed_redis_failures.get() == swallowed_before: - breaker.record_success() + if _swallowed_redis_failures.get() != admission.swallowed_before: + return + if breaker.is_half_open() and not admission.is_probe: + return + breaker.record_success() async def _run_under_circuit_breaker( @@ -432,14 +447,14 @@ async def _run_under_circuit_breaker( Shared by the method decorator and the Lua script executor so both feed the same health signal. """ - swallowed_before: Final = _enter_circuit_breaker(breaker, name) + admission: Final = _enter_circuit_breaker(breaker, name) try: result: Final = await call() except Exception as e: if _is_redis_health_failure(e): breaker.record_failure(is_timeout=_is_redis_timeout_failure(e)) raise - _exit_circuit_breaker(breaker, swallowed_before) + _exit_circuit_breaker(breaker, admission) return result @@ -449,14 +464,14 @@ def _run_under_circuit_breaker_sync( call: Callable[[], _RedisCallResult], ) -> _RedisCallResult: """Run one blocking Redis call under a circuit breaker, feeding the same health signal as the async path.""" - swallowed_before: Final = _enter_circuit_breaker(breaker, name) + admission: Final = _enter_circuit_breaker(breaker, name) try: result: Final = call() except Exception as e: if _is_redis_health_failure(e): breaker.record_failure(is_timeout=_is_redis_timeout_failure(e)) raise - _exit_circuit_breaker(breaker, swallowed_before) + _exit_circuit_breaker(breaker, admission) return result @@ -1395,12 +1410,12 @@ class RedisCache(BaseCache): key_value_dict = {} _key_list: Final = [key for key in key_list if key is not None] start_time: Final = time.time() - swallowed_before: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache") + admission: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache") try: _keys: Final = [self.check_and_fix_namespace(key=cache_key or "") for cache_key in _key_list] results: Final = self._run_redis_mget_operation(keys=_keys) - _exit_circuit_breaker(self._circuit_breaker, swallowed_before) + _exit_circuit_breaker(self._circuit_breaker, admission) end_time: Final = time.time() _duration: Final = end_time - start_time self.service_logger_obj.service_success_hook( diff --git a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py index 4be1331e955..bc67617e444 100644 --- a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py +++ b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py @@ -1,10 +1,11 @@ import asyncio import json +import logging from typing import TYPE_CHECKING, Any, Final from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid -from litellm.caching.redis_cache import RedisCache +from litellm.caching.redis_cache import RedisCache, log_redis_failure from litellm.constants import DEFAULT_CRON_JOB_LOCK_TTL_SECONDS from litellm.proxy.db.db_transaction_queue.base_update_queue import service_logger_obj from litellm.types.services import ServiceTypes @@ -109,7 +110,7 @@ end ) return False except Exception as e: - verbose_proxy_logger.error("Error acquiring Redis lock for %s: %s", cronjob_id, e) + log_redis_failure(verbose_proxy_logger, logging.ERROR, f"Error acquiring Redis lock for {cronjob_id}", e) return False async def release_lock( @@ -151,7 +152,7 @@ end cronjob_id, ) except Exception as e: - verbose_proxy_logger.error("Error releasing Redis lock for %s: %s", cronjob_id, e) + log_redis_failure(verbose_proxy_logger, logging.ERROR, f"Error releasing Redis lock for {cronjob_id}", e) async def _compare_and_delete_lock(self, lock_key: str) -> int: """ diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index c6f0caa1dca..9df9c44a30f 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -1103,3 +1103,52 @@ def test_recovery_probe_still_closes_the_breaker(): assert breaker._state == breaker.CLOSED assert breaker.is_open() is False + + +@pytest.mark.asyncio +async def test_stale_success_during_the_recovery_probe_leaves_the_breaker_to_the_probe(): + """A call admitted before the trip that finishes while HALF_OPEN must not close the breaker. + + Only the one call designated as the recovery probe has actually reached Redis after the + outage, so closing on the straggler's success resumed full Redis traffic before the probe + had proven anything. + """ + from litellm.caching.redis_cache import RedisCircuitBreaker, _run_under_circuit_breaker + + breaker = RedisCircuitBreaker(failure_threshold=3, recovery_timeout=60) + stale_admitted = asyncio.Event() + stale_release = asyncio.Event() + probe_admitted = asyncio.Event() + probe_release = asyncio.Event() + + async def stale_call() -> str: + stale_admitted.set() + await stale_release.wait() + return "stale" + + async def probe_call() -> str: + probe_admitted.set() + await probe_release.wait() + return "probe" + + stale = asyncio.ensure_future(_run_under_circuit_breaker(breaker, "op", stale_call)) + await stale_admitted.wait() + for _ in range(3): + breaker.record_failure() + assert breaker._state == breaker.OPEN + breaker._opened_at = time.time() - 9999 + probe = asyncio.ensure_future(_run_under_circuit_breaker(breaker, "op", probe_call)) + await probe_admitted.wait() + assert breaker._state == breaker.HALF_OPEN + + stale_release.set() + assert await stale == "stale" + + assert breaker._state == breaker.HALF_OPEN, "the straggler must not close the breaker for the probe" + assert breaker.is_open() is True + + probe_release.set() + assert await probe == "probe" + + assert breaker._state == breaker.CLOSED + assert breaker.is_open() is False diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_pod_lock_manager.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_pod_lock_manager.py index ecd5c5f50c0..4684c3213d6 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_pod_lock_manager.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_pod_lock_manager.py @@ -1,4 +1,5 @@ import json +import logging from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock, patch @@ -6,6 +7,7 @@ import pytest from fastapi.testclient import TestClient +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.constants import DEFAULT_CRON_JOB_LOCK_TTL_SECONDS from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager @@ -215,6 +217,22 @@ async def test_redis_error_handling(pod_lock_manager, mock_redis): ) +@pytest.mark.asyncio +async def test_lock_refused_by_the_open_circuit_breaker_is_not_logged_as_an_error(pod_lock_manager, mock_redis, caplog): + """Every cron job retries its lock on a timer, so an open breaker must not add an error line per cycle.""" + refused = RedisCircuitBreakerOpenError("Redis circuit breaker is open - skipping async_set_cache") + mock_redis.async_set_cache.side_effect = refused + mock_redis.async_get_cache.return_value = pod_lock_manager.pod_id + mock_redis.async_delete_cache.side_effect = refused + + with caplog.at_level(logging.ERROR): + acquired = await pod_lock_manager.acquire_lock(cronjob_id="test_job") + await pod_lock_manager.release_lock(cronjob_id="test_job") + + assert acquired is False + assert caplog.records == [] + + @pytest.mark.asyncio async def test_bytes_handling(pod_lock_manager, mock_redis): """ From 880ccc76a5bcc89b07a7ca337662a902bf56083e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:37:39 -0700 Subject: [PATCH 112/126] fix(streaming): keep admitted mock streams alive with empty stream_options and honor zero prompt counts (#40650) * fix(streaming): keep usage-only chunks from crashing streams with empty stream_options The usage-only chunk branch in CustomStreamWrapper.chunk_creator indexed stream_options["include_usage"] directly, so a caller passing stream_options={} hit a KeyError that surfaced as MidStreamFallbackError. Streaming mock_response with an admission input_tokens count (#40637) now always emits such a chunk, which made the crash reachable. Reuse the send_stream_usage policy computed at init instead. Also annotate the #40637 test bindings with Final. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(streaming): report admitted zero prompt tokens instead of recounting in mock streams Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/streaming_handler.py | 7 +- litellm/main.py | 9 ++ tests/test_litellm/test_main.py | 143 ++++++++++++++---- 3 files changed, 131 insertions(+), 28 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index db23929e0c3..6a5a8832cc6 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -236,9 +236,11 @@ class CustomStreamWrapper: stream_options=None, make_call: Callable | None = None, _response_headers: dict | httpx.Headers | None = None, + count_prompt_tokens: Callable[[], int] | None = None, ): self.model = model self.make_call = make_call + self.count_prompt_tokens = count_prompt_tokens self.custom_llm_provider = custom_llm_provider self.logging_obj: LiteLLMLoggingObject = logging_obj self.completion_stream = completion_stream @@ -1641,7 +1643,7 @@ class CustomStreamWrapper: except Exception: model_response.choices[0].delta = Delta() else: - if self.stream_options is not None and self.stream_options["include_usage"] is True: + if self.send_stream_usage is True: model_response.choices = [] return model_response self._record_usage_only_chunk(model_response=model_response) @@ -1996,6 +1998,7 @@ class CustomStreamWrapper: chunks=self.chunks, messages=self.messages, logging_obj=self.logging_obj, + count_prompt_tokens=self.count_prompt_tokens, ) except Exception as e: # stream_chunk_builder can re-raise (as APIError) on large agentic @@ -2248,6 +2251,7 @@ class CustomStreamWrapper: chunks=self.chunks, messages=self.messages, logging_obj=self.logging_obj, + count_prompt_tokens=self.count_prompt_tokens, ) except Exception as e: # see sync __next__: a raise from stream_chunk_builder inside this @@ -2371,6 +2375,7 @@ class CustomStreamWrapper: chunks=self.chunks, messages=self.messages if isinstance(self.messages, list) else None, logging_obj=self.logging_obj, + count_prompt_tokens=self.count_prompt_tokens, ) if partial_response is None: return diff --git a/litellm/main.py b/litellm/main.py index 10fb32828f1..17edafcdfca 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -850,6 +850,12 @@ def admission_input_tokens(kwargs: Mapping[str, object]) -> int | None: ) +def admitted_prompt_token_counter(prompt_tokens: int | None) -> Callable[[], int] | None: + if prompt_tokens is None: + return None + return lambda: prompt_tokens + + def mock_completion( model: str, messages: list, @@ -935,6 +941,7 @@ def mock_completion( if stream is True: model_response = ModelResponseStream() + count_prompt_tokens: Final = admitted_prompt_token_counter(prompt_tokens) # don't try to access stream object, if kwargs.get("acompletion", False) is True: return CustomStreamWrapper( @@ -944,6 +951,7 @@ def mock_completion( model=model, custom_llm_provider="openai", logging_obj=logging, + count_prompt_tokens=count_prompt_tokens, ) return CustomStreamWrapper( completion_stream=mock_completion_streaming_obj( @@ -952,6 +960,7 @@ def mock_completion( model=model, custom_llm_provider="openai", logging_obj=logging, + count_prompt_tokens=count_prompt_tokens, ) if isinstance(mock_response, litellm.MockException): raise mock_response diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index a36ca229981..f71225c6fc5 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -2422,9 +2422,13 @@ def test_mock_completion_usage_falls_back_to_default_without_admission_count(): _ADMISSION_INPUT_TOKENS: Final = 51234 -_ADMISSION_METADATA: Final = { - "user_api_key_budget_reservation": {"reserved_cost": 1.0, "input_tokens": _ADMISSION_INPUT_TOKENS} -} + + +def _admission_metadata(input_tokens: int) -> dict[str, object]: # mutable-ok: logging writes into metadata + return {"user_api_key_budget_reservation": {"reserved_cost": 1.0, "input_tokens": input_tokens}} + + +_ADMISSION_METADATA: Final = _admission_metadata(_ADMISSION_INPUT_TOKENS) _MOCK_STREAM_MESSAGES: Final = [{"role": "user", "content": "hello " * 200}] _STREAM_CHUNK_BUILDER_TOKEN_COUNTER: Final = "litellm.litellm_core_utils.streaming_chunk_builder_utils.token_counter" @@ -2440,7 +2444,7 @@ def _client_usage_chunks(chunks: list[ModelResponseStream]) -> list[Usage]: @pytest.mark.parametrize("n", (None, 2)) def test_mock_completion_stream_usage_reports_admission_input_tokens_without_tokenizer_fallback(n: int | None): with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: - chunks = list( + chunks: Final = list( litellm.completion( model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES, @@ -2453,7 +2457,7 @@ def test_mock_completion_stream_usage_reports_admission_input_tokens_without_tok ) ) - usage_chunks = _client_usage_chunks(chunks) + usage_chunks: Final = _client_usage_chunks(chunks) assert len(usage_chunks) == 1 assert usage_chunks[0].prompt_tokens == _ADMISSION_INPUT_TOKENS assert usage_chunks[0].completion_tokens == litellm_main.DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT @@ -2469,7 +2473,7 @@ async def test_mock_acompletion_stream_usage_reports_admission_input_tokens_with n: int | None, ): with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: - response = await litellm.acompletion( + response: Final = await litellm.acompletion( model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES, mock_response="ok", @@ -2479,9 +2483,9 @@ async def test_mock_acompletion_stream_usage_reports_admission_input_tokens_with stream_options={"include_usage": True}, litellm_metadata=_ADMISSION_METADATA, ) - chunks = [chunk async for chunk in response] + chunks: Final = [chunk async for chunk in response] - usage_chunks = _client_usage_chunks(chunks) + usage_chunks: Final = _client_usage_chunks(chunks) assert len(usage_chunks) == 1 assert usage_chunks[0].prompt_tokens == _ADMISSION_INPUT_TOKENS assert usage_chunks[0].total_tokens == _ADMISSION_INPUT_TOKENS + usage_chunks[0].completion_tokens @@ -2492,7 +2496,7 @@ async def test_mock_acompletion_stream_usage_reports_admission_input_tokens_with def test_mock_completion_stream_without_include_usage_hides_usage_chunk_but_logs_admission_count(): with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: - chunks = list( + chunks: Final = list( litellm.completion( model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES, @@ -2509,10 +2513,48 @@ def test_mock_completion_stream_without_include_usage_hides_usage_chunk_but_logs assert _prompt_token_counter_calls(token_counter) == [] +def test_mock_completion_stream_with_empty_stream_options_completes_and_logs_admission_count(): + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + chunks: Final = list( + litellm.completion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + stream_options={}, + metadata=_ADMISSION_METADATA, + ) + ) + + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "ok" + assert _client_usage_chunks(chunks) == [] + assert _prompt_token_counter_calls(token_counter) == [] + + +@pytest.mark.asyncio +async def test_mock_acompletion_stream_with_empty_stream_options_completes_and_logs_admission_count(): + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + response: Final = await litellm.acompletion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + stream_options={}, + litellm_metadata=_ADMISSION_METADATA, + ) + chunks: Final = [chunk async for chunk in response] + + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "ok" + assert _client_usage_chunks(chunks) == [] + assert _prompt_token_counter_calls(token_counter) == [] + + def test_mock_completion_stream_without_admission_count_falls_back_to_tokenizer(): expected_prompt_tokens: Final = litellm.token_counter(model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES) with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: - chunks = list( + chunks: Final = list( litellm.completion( model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES, @@ -2524,7 +2566,7 @@ def test_mock_completion_stream_without_admission_count_falls_back_to_tokenizer( ) ) - usage_chunks = _client_usage_chunks(chunks) + usage_chunks: Final = _client_usage_chunks(chunks) assert len(usage_chunks) == 1 assert usage_chunks[0].prompt_tokens == expected_prompt_tokens assert usage_chunks[0].total_tokens == expected_prompt_tokens + usage_chunks[0].completion_tokens @@ -2535,7 +2577,7 @@ def test_mock_completion_stream_without_admission_count_falls_back_to_tokenizer( async def test_mock_acompletion_stream_without_admission_count_falls_back_to_tokenizer(): expected_prompt_tokens: Final = litellm.token_counter(model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES) with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: - response = await litellm.acompletion( + response: Final = await litellm.acompletion( model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES, mock_response="ok", @@ -2543,40 +2585,87 @@ async def test_mock_acompletion_stream_without_admission_count_falls_back_to_tok stream=True, stream_options={"include_usage": True}, ) - chunks = [chunk async for chunk in response] + chunks: Final = [chunk async for chunk in response] - usage_chunks = _client_usage_chunks(chunks) + usage_chunks: Final = _client_usage_chunks(chunks) assert len(usage_chunks) == 1 assert usage_chunks[0].prompt_tokens == expected_prompt_tokens assert len(_prompt_token_counter_calls(token_counter)) >= 1 -def test_mock_completion_stream_and_non_stream_report_the_same_admission_usage(): - non_stream = litellm.completion( +def _usage_triple(usage: Usage) -> tuple[int, int, int]: + return (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) + + +@pytest.mark.parametrize("input_tokens", (_ADMISSION_INPUT_TOKENS, 0)) +def test_mock_completion_stream_and_non_stream_report_the_same_admission_usage(input_tokens: int): + metadata: Final = _admission_metadata(input_tokens) + non_stream: Final = litellm.completion( model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES, mock_response="ok", api_key="mock", - metadata=_ADMISSION_METADATA, + metadata=metadata, ) - chunks = list( - litellm.completion( + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + chunks: Final = list( + litellm.completion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + stream_options={"include_usage": True}, + metadata=metadata, + ) + ) + + assert _usage_triple(non_stream.usage) == _usage_triple(_client_usage_chunks(chunks)[0]) + assert non_stream.usage.prompt_tokens == input_tokens + assert _prompt_token_counter_calls(token_counter) == [] + + +@pytest.mark.asyncio +async def test_mock_acompletion_stream_reports_zero_admission_input_tokens_without_tokenizer_fallback(): + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + response: Final = await litellm.acompletion( model="openai/gpt-5.4-mini", - messages=_MOCK_STREAM_MESSAGES, + messages=[{"role": "user", "content": ""}], mock_response="ok", api_key="mock", stream=True, stream_options={"include_usage": True}, - metadata=_ADMISSION_METADATA, + litellm_metadata=_admission_metadata(0), + ) + chunks: Final = [chunk async for chunk in response] + + usage_chunks: Final = _client_usage_chunks(chunks) + assert len(usage_chunks) == 1 + assert _usage_triple(usage_chunks[0]) == (0, usage_chunks[0].completion_tokens, usage_chunks[0].completion_tokens) + assert _prompt_token_counter_calls(token_counter) == [] + + +def test_mock_text_completion_stream_and_non_stream_report_the_same_zero_admission_usage(): + metadata: Final = _admission_metadata(0) + non_stream: Final = litellm.text_completion( + model="openai/gpt-5.4-mini", prompt="", mock_response="ok", api_key="mock", metadata=metadata + ) + chunks: Final = list( + litellm.text_completion( + model="openai/gpt-5.4-mini", + prompt="", + mock_response="ok", + api_key="mock", + stream=True, + stream_options={"include_usage": True}, + metadata=metadata, ) ) - stream_usage: Final = _client_usage_chunks(chunks)[0] - assert (non_stream.usage.prompt_tokens, non_stream.usage.completion_tokens, non_stream.usage.total_tokens) == ( - stream_usage.prompt_tokens, - stream_usage.completion_tokens, - stream_usage.total_tokens, - ) + stream_usages: Final = tuple(chunk.usage for chunk in chunks if getattr(chunk, "usage", None) is not None) + assert len(stream_usages) == 1 + assert _usage_triple(non_stream.usage) == _usage_triple(stream_usages[0]) + assert non_stream.usage.prompt_tokens == 0 def test_mock_completion_stream_with_model_response(): From 56e2d8846da375ae157e27d1a789030882066f2d Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 18:08:13 -0700 Subject: [PATCH 113/126] feat(ui): link the entity cells on the team detail page's keys table The team detail page's Virtual Keys table showed Organization ID, User Email, User ID and Created By as dead text, so getting from a key to the org or user behind it meant copying an id and searching for it. Those four cells now render as links, reusing the sentinel-aware href helpers, so default_user_id and the litellm-dashboard team stay plain text instead of pointing at pages that do not exist. The Created By cell was a verbatim copy of the Virtual Keys page's user popover, so that moved into the shared table_cells kit and both tables now use the one implementation. Claude-Session: https://claude.ai/code/session_01NfwfQhamRNnSqgXMUjf3h4 --- .../VirtualKeysPage/keyTableColumns.tsx | 62 +----------- .../shared/table_cells/UserPopoverCell.tsx | 62 ++++++++++++ .../components/shared/table_cells/index.ts | 1 + .../team/TeamVirtualKeysTable.test.tsx | 62 ++++++++++++ .../components/team/TeamVirtualKeysTable.tsx | 94 ++++++++----------- 5 files changed, 168 insertions(+), 113 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx index 9dd4b1c2d59..6eea77ae827 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx @@ -9,17 +9,17 @@ import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/h import { Skeleton } from "@/components/ui/skeleton"; import { DateCell, + ENTITY_CELL_TITLE_CLASSES, IdCell, IdentityCell, ModelsCell, SpendBudgetCell, StatusBadge, + UserPopoverCell, type StatusTone, } from "@/components/shared/table_cells"; -import { orgDetailHref, teamDetailHref, userDetailHref } from "@/utils/entityLinks"; -import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; +import { orgDetailHref, teamDetailHref } from "@/utils/entityLinks"; -import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; import { KeyResponse, Team } from "../key_team_helpers/key_list"; import { Organization } from "../networking"; @@ -29,8 +29,6 @@ interface KeyStatus { tooltip?: string; } -const ENTITY_CELL_TITLE_CLASSES = "font-mono text-xs font-normal"; - const SPEND_BUDGET_SORT_FIELDS: DataTableSortField[] = [ { id: "spend", label: "Spend" }, { id: "max_budget", label: "Budget" }, @@ -66,60 +64,6 @@ const getKeyStatus = (key: KeyResponse): KeyStatus => { }; }; -const UserPopoverCell = ({ - userAlias, - userEmail, - userId, - width, -}: { - userAlias: string | null; - userEmail: string | null; - userId: string | null; - width: number; -}) => { - const displayValue = userAlias || userEmail || userId; - const isDefaultAdmin = userId === DEFAULT_PROXY_ADMIN_USER_ID; - - const popoverContent = ( -
- {[ - { label: "User Alias", value: userAlias }, - { label: "User Email", value: userEmail }, - { label: "User ID", value: userId }, - ].map(({ label, value }) => ( -
- {label} - {value ? ( - - ) : ( - - - )} -
- ))} -
- ); - - const trigger = - isDefaultAdmin && !userAlias && !userEmail ? ( - - ) : ( - - ); - - return ( - - }> - {trigger} - - {popoverContent} - - ); -}; - const InfoHeader = ({ label, tooltip }: { label: string; tooltip: string }) => ( {label} diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx new file mode 100644 index 00000000000..223d66d7421 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx @@ -0,0 +1,62 @@ +"use client"; + +import DefaultProxyAdminTag from "@/components/common_components/DefaultProxyAdminTag"; +import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; +import { userDetailHref } from "@/utils/entityLinks"; +import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; + +import { IdCell } from "./id_cell"; +import { IdentityCell } from "./identity_cell"; + +export const ENTITY_CELL_TITLE_CLASSES = "font-mono text-xs font-normal"; + +interface UserPopoverCellProps { + userAlias: string | null; + userEmail: string | null; + userId: string | null; + width: number; +} + +export function UserPopoverCell({ userAlias, userEmail, userId, width }: UserPopoverCellProps) { + const displayValue = userAlias || userEmail || userId; + const isDefaultAdmin = userId === DEFAULT_PROXY_ADMIN_USER_ID; + + const popoverContent = ( +
+ {[ + { label: "User Alias", value: userAlias }, + { label: "User Email", value: userEmail }, + { label: "User ID", value: userId }, + ].map(({ label, value }) => ( +
+ {label} + {value ? ( + + ) : ( + - + )} +
+ ))} +
+ ); + + const trigger = + isDefaultAdmin && !userAlias && !userEmail ? ( + + ) : ( + + ); + + return ( + + }> + {trigger} + + {popoverContent} + + ); +} diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/index.ts b/ui/litellm-dashboard/src/components/shared/table_cells/index.ts index 99b54f9aad0..34e3701eed4 100644 --- a/ui/litellm-dashboard/src/components/shared/table_cells/index.ts +++ b/ui/litellm-dashboard/src/components/shared/table_cells/index.ts @@ -13,3 +13,4 @@ export { ModelsCell } from "./models_cell"; export { MoneyCell } from "./money_cell"; export { SpendBudgetCell } from "./spend_budget_cell"; export { StatusBadge, type StatusTone } from "./status_badge"; +export { UserPopoverCell, ENTITY_CELL_TITLE_CLASSES } from "./UserPopoverCell"; diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx index 0d9d0988aa1..76df0d12539 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx @@ -10,6 +10,8 @@ vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({ useKeys: vi.fn(), })); +vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn() }) })); + vi.mock("../key_team_helpers/fetch_available_models_team_key", () => ({ getModelDisplayName: vi.fn((model: string) => model), })); @@ -384,4 +386,64 @@ describe("TeamVirtualKeysTable", () => { expect(screen.getByText("Key Info View")).toBeInTheDocument(); }); }); + + describe("entity links out of the key rows", () => { + const renderRow = async (key: KeyResponse, organization: Organization | null = null) => { + mockUseKeys.mockReturnValue({ + data: { keys: [key], total_count: 1, current_page: 1, total_pages: 1 } as KeysResponse, + isPending: false, + isFetching: false, + refetch: vi.fn(), + } as any); + renderWithProviders(); + return (await screen.findByText(key.key_alias as string)).closest("tr") as HTMLElement; + }; + + it("points the Organization ID cell at the org's detail page", async () => { + const row = await renderRow(createMockKey({ organization_id: null }), mockOrganization); + expect(within(row).getByRole("link", { name: "org-123" })).toHaveAttribute( + "href", + "/ui/organizations?org=org-123", + ); + }); + + it("points the User Email and User ID cells at the owning user's detail page", async () => { + const row = await renderRow( + createMockKey({ user_id: "user-1", user: { user_id: "user-1", user_email: "alice@example.com" } }), + ); + expect(within(row).getByRole("link", { name: "alice@example.com" })).toHaveAttribute( + "href", + "/ui/users?user=user-1", + ); + expect(within(row).getByRole("link", { name: "user-1" })).toHaveAttribute("href", "/ui/users?user=user-1"); + }); + + it("points the Created By cell at the creator's detail page", async () => { + const row = await renderRow( + createMockKey({ + created_by: "creator-1", + created_by_user: { user_id: "creator-1", user_email: "creator@example.com", user_alias: "The Creator" }, + }), + ); + expect(within(row).getByRole("link", { name: "The Creator" })).toHaveAttribute( + "href", + "/ui/users?user=creator-1", + ); + }); + + it("leaves the default_user_id placeholder unlinked in the User ID and Created By cells", async () => { + const placeholder = { user_id: "default_user_id", user_email: "admin@example.com", user_alias: "Proxy Admin" }; + const row = await renderRow( + createMockKey({ + user_id: placeholder.user_id, + user: placeholder, + created_by: placeholder.user_id, + created_by_user: placeholder, + }), + ); + expect(within(row).getByText("Default Proxy Admin")).toBeInTheDocument(); + expect(within(row).getByText("Proxy Admin")).toBeInTheDocument(); + expect(within(row).queryByRole("link", { name: "Proxy Admin" })).not.toBeInTheDocument(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx index b0380255b95..5b1b71e060e 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx @@ -1,8 +1,14 @@ "use client"; import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; import { SimpleTooltip } from "@/components/ui/tooltip"; -import CopyButton from "@/components/shared/CopyButton"; -import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; +import { + DateCell, + ENTITY_CELL_TITLE_CLASSES, + IdCell, + IdentityCell, + MoneyCell, + UserPopoverCell, +} from "@/components/shared/table_cells"; import { DataTable, DataTableFilterDrawer, @@ -11,8 +17,9 @@ import { DataTableToolbar, } from "@/components/shared/DataTable"; import { Badge } from "@/components/ui/badge"; -import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; import { Input } from "@/components/ui/input"; +import { orgDetailHref, userDetailHref } from "@/utils/entityLinks"; +import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; import { useDebouncedValue } from "@tanstack/react-pacer/debouncer"; import { ColumnDef, ColumnFiltersState, OnChangeFn, PaginationState, SortingState } from "@tanstack/react-table"; @@ -168,7 +175,15 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi header: "Organization ID", size: 140, enableSorting: false, - cell: (info) => (info.getValue() ? info.renderValue() : "-"), + cell: (info) => { + const orgId = info.getValue() as string | null; + if (!orgId) return "-"; + return ( + + + + ); + }, }, { id: "user_email", @@ -179,9 +194,14 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi cell: (info) => { const user = info.getValue() as { user_email?: string } | undefined; const value = user?.user_email; + const userId = info.row.original.user_id; return ( - {value ?? "-"} + ); }, @@ -194,10 +214,16 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi enableSorting: false, cell: (info) => { const userId = info.getValue() as string | null; - const displayValue = userId === "default_user_id" ? "Default Proxy Admin" : userId; + if (userId === DEFAULT_PROXY_ADMIN_USER_ID) { + return ; + } return ( - - {displayValue ?? "-"} + + ); }, @@ -221,53 +247,13 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi const userId = info.getValue() as string | null; if (!userId) return "-"; const { created_by_user } = info.row.original; - const userAlias = created_by_user?.user_alias ?? null; - const userEmail = created_by_user?.user_email ?? null; - const isDefaultAdmin = userId === "default_user_id"; - const displayValue = userAlias || userEmail || userId; - - const popoverContent = ( -
- {[ - { label: "User Alias", value: userAlias }, - { label: "User Email", value: userEmail }, - { label: "User ID", value: userId }, - ].map(({ label, value }) => ( -
- {label} - {value ? ( - - {value} - - - ) : ( - - - )} -
- ))} -
- ); - - if (isDefaultAdmin && !userAlias && !userEmail) { - return ( - - }> - - - {popoverContent} - - ); - } - return ( - - } - > - {displayValue} - - {popoverContent} - + ); }, }, From 06b259e0920792600c35c9951297fb83cd236c35 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 18:31:19 -0700 Subject: [PATCH 114/126] fix(ui): name the popover copy buttons after the field they copy The shared user popover copied alias, email and ID through three copy buttons that all announced themselves as "Copy ID", so a screen reader could not tell them apart. IdCell now takes the label, defaulting to the old text everywhere else. Also drops the closest("tr") the new link tests used, which put the testing-library/no-node-access budget over its ceiling, and asserts the sentinel row leaves User Email and the admin badge unlinked too. Claude-Session: https://claude.ai/code/session_01NfwfQhamRNnSqgXMUjf3h4 --- .../shared/table_cells/UserPopoverCell.tsx | 2 +- .../shared/table_cells/id_cell.test.tsx | 8 ++++++++ .../components/shared/table_cells/id_cell.tsx | 4 +++- .../team/TeamVirtualKeysTable.test.tsx | 20 ++++++++++--------- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx index 223d66d7421..eb786e45541 100644 --- a/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx +++ b/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx @@ -31,7 +31,7 @@ export function UserPopoverCell({ userAlias, userEmail, userId, width }: UserPop
{label} {value ? ( - + ) : ( - )} diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx index 395c1815dd0..c715210fdde 100644 --- a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx @@ -71,6 +71,14 @@ describe("IdCell", () => { expect(rowClick).not.toHaveBeenCalled(); }); + it("names the copy button after the field it copies", async () => { + const user = userEvent.setup(); + render(); + expect(screen.queryByRole("button", { name: "Copy ID" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Copy User Email" })); + expect(copyToClipboardMock).toHaveBeenCalledWith("alice@example.com"); + }); + it("passes dataTestId through to the id element", () => { render(); expect(screen.getByTestId("key-id-cell")).toHaveTextContent("k-1"); diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx index 1b109a86106..33c7f835e64 100644 --- a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx +++ b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx @@ -15,6 +15,7 @@ interface IdCellProps { variant?: IdCellVariant; onClick?: (value: string) => void; copyable?: boolean; + copyLabel?: string; truncate?: boolean; fallback?: string; tooltip?: React.ReactNode; @@ -39,6 +40,7 @@ export function IdCell({ variant = "pill", onClick, copyable = false, + copyLabel = "Copy ID", truncate = true, fallback = "-", tooltip, @@ -80,7 +82,7 @@ export function IdCell({ {withTooltip}