From 691ca03266539c7278c88a4d10eb161fc7e68e2b Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 13:20:31 -0700 Subject: [PATCH 01/40] fix(sdk): resolve litellm.proxy lazily on attribute access (#45845) The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main --- litellm/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index 4196912488f..a6bbd132cbf 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -2386,6 +2386,11 @@ def __getattr__(name: str) -> Any: handler_func: Final = registry[name] return handler_func(name) + if name == "proxy": + import importlib + + return importlib.import_module("litellm.proxy") + if name == "harness" or name in _AGENT_EXPORTS: import importlib From c397b57f2b8685d05e7c5ffc5bdacef6a310f33f Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 13:32:34 -0700 Subject: [PATCH 02/40] refactor(anthropic): move prompt cache pricing helper out of the proxy package (#45753) * refactor(anthropic): move prompt cache pricing helper out of the proxy package Part of LIT-9423 * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/anthropic/cache_aware_routing.py | 2 +- .../llms/anthropic/prompt_cache_pricing.py | 100 +++++++++++++++++ .../common_utils/prompt_cache_pricing.py | 101 +----------------- .../test_anthropic_prompt_cache_pricing.py} | 2 +- 4 files changed, 104 insertions(+), 101 deletions(-) create mode 100644 litellm/llms/anthropic/prompt_cache_pricing.py rename tests/unit/{proxy/common_utils/test_prompt_cache_pricing.py => llms/anthropic/test_anthropic_prompt_cache_pricing.py} (97%) diff --git a/litellm/llms/anthropic/cache_aware_routing.py b/litellm/llms/anthropic/cache_aware_routing.py index f049e4ac187..515373448f4 100644 --- a/litellm/llms/anthropic/cache_aware_routing.py +++ b/litellm/llms/anthropic/cache_aware_routing.py @@ -24,7 +24,7 @@ from litellm.llms.anthropic.prompt_cache_prediction import ( resolve_prediction_target, supported_prediction_headers, ) -from litellm.proxy.common_utils.prompt_cache_pricing import price_cache_tokens +from litellm.llms.anthropic.prompt_cache_pricing import price_cache_tokens from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.management_endpoints.prompt_cache_prediction import ( CacheCostScenario, diff --git a/litellm/llms/anthropic/prompt_cache_pricing.py b/litellm/llms/anthropic/prompt_cache_pricing.py new file mode 100644 index 00000000000..b07873db510 --- /dev/null +++ b/litellm/llms/anthropic/prompt_cache_pricing.py @@ -0,0 +1,100 @@ +from collections.abc import Mapping +from datetime import datetime, timezone +from math import isfinite +from typing import Final + +from pydantic import TypeAdapter + +import litellm +from litellm.cost_calculator import ( + completion_cost, # pyright: ignore[reportUnknownVariableType] # legacy optional parameters are untyped + select_model_name_for_cost_calc, +) +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.types.management_endpoints.prompt_cache_prediction import CacheTokenBuckets +from litellm.types.utils import CacheCreationTokenDetails, ModelResponse, PromptTokensDetailsWrapper, Usage + +_PRICE_ENTRY: Final = TypeAdapter(Mapping[str, object]) + + +def _valid_price(value: object) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and isfinite(value) and value >= 0 + + +def _has_required_prices(prices: Mapping[str, object], tokens: CacheTokenBuckets, completion_tokens: int = 0) -> bool: + required: Final = ( + ("input_cost_per_token", True), + ("cache_read_input_token_cost", tokens.cache_read_input_tokens > 0), + ("cache_creation_input_token_cost", tokens.cache_creation_5m_input_tokens > 0), + ("cache_creation_input_token_cost_above_1hr", tokens.cache_creation_1h_input_tokens > 0), + ("output_cost_per_token", completion_tokens > 0), + ) + if any(needed and not _valid_price(prices.get(key)) for key, needed in required): + return False + return all( + _valid_price(value) + for key, value in prices.items() + if value is not None and any(needed and key.startswith(f"{base}_above_") for base, needed in required) + ) + + +def price_cache_tokens( + model: str, deployment_id: str, tokens: CacheTokenBuckets, completion_tokens: int = 0 +) -> float | None: + try: + selected_model: Final = select_model_name_for_cost_calc( + model=model, + completion_response=None, + custom_pricing=True, + custom_llm_provider="anthropic", + router_model_id=deployment_id, + ) + if selected_model is None: + return None + model_info: Final = litellm.get_model_info(model=selected_model, custom_llm_provider="anthropic") + registry: Final = _PRICE_ENTRY.validate_python(litellm.model_cost) # pyright: ignore[reportUnknownMemberType] # legacy registry is validated at this boundary + price_entry: Final = registry.get(model_info["key"]) + if price_entry is None: + return None + prices: Final = _PRICE_ENTRY.validate_python(price_entry) + if completion_tokens < 0 or not _has_required_prices(prices, tokens, completion_tokens): + return None + usage: Final = Usage( + prompt_tokens=tokens.total_tokens, + completion_tokens=completion_tokens, + total_tokens=tokens.total_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=tokens.cache_read_input_tokens, + cache_creation_tokens=tokens.cache_creation_5m_input_tokens + tokens.cache_creation_1h_input_tokens, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_5m_input_tokens=tokens.cache_creation_5m_input_tokens, + ephemeral_1h_input_tokens=tokens.cache_creation_1h_input_tokens, + ), + ), + ) + logging_obj: Final = Logging( + model=model, + messages=[], + stream=False, + call_type="completion", + start_time=datetime.now(timezone.utc), + litellm_call_id="prompt-cache-prediction", + function_id="prompt-cache-prediction", + ) + completion_cost( + completion_response=ModelResponse(model=model, usage=usage), + model=model, + custom_llm_provider="anthropic", + custom_pricing=True, + router_model_id=deployment_id, + litellm_logging_obj=logging_obj, + ) + breakdown: Final = logging_obj.cost_breakdown + input_cost: Final = breakdown.get("input_cost") if breakdown is not None else None + output_cost: Final = breakdown.get("output_cost") if breakdown is not None else None + if input_cost is None or output_cost is None: + return None + cost: Final = input_cost + output_cost + return cost if _valid_price(cost) else None + except Exception: # noqa: BLE001 # the shared pricing owners raise plain Exception for unpriceable models + return None diff --git a/litellm/proxy/common_utils/prompt_cache_pricing.py b/litellm/proxy/common_utils/prompt_cache_pricing.py index b07873db510..55682ba872c 100644 --- a/litellm/proxy/common_utils/prompt_cache_pricing.py +++ b/litellm/proxy/common_utils/prompt_cache_pricing.py @@ -1,100 +1,3 @@ -from collections.abc import Mapping -from datetime import datetime, timezone -from math import isfinite -from typing import Final +from litellm.llms.anthropic.prompt_cache_pricing import price_cache_tokens -from pydantic import TypeAdapter - -import litellm -from litellm.cost_calculator import ( - completion_cost, # pyright: ignore[reportUnknownVariableType] # legacy optional parameters are untyped - select_model_name_for_cost_calc, -) -from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.types.management_endpoints.prompt_cache_prediction import CacheTokenBuckets -from litellm.types.utils import CacheCreationTokenDetails, ModelResponse, PromptTokensDetailsWrapper, Usage - -_PRICE_ENTRY: Final = TypeAdapter(Mapping[str, object]) - - -def _valid_price(value: object) -> bool: - return isinstance(value, (int, float)) and not isinstance(value, bool) and isfinite(value) and value >= 0 - - -def _has_required_prices(prices: Mapping[str, object], tokens: CacheTokenBuckets, completion_tokens: int = 0) -> bool: - required: Final = ( - ("input_cost_per_token", True), - ("cache_read_input_token_cost", tokens.cache_read_input_tokens > 0), - ("cache_creation_input_token_cost", tokens.cache_creation_5m_input_tokens > 0), - ("cache_creation_input_token_cost_above_1hr", tokens.cache_creation_1h_input_tokens > 0), - ("output_cost_per_token", completion_tokens > 0), - ) - if any(needed and not _valid_price(prices.get(key)) for key, needed in required): - return False - return all( - _valid_price(value) - for key, value in prices.items() - if value is not None and any(needed and key.startswith(f"{base}_above_") for base, needed in required) - ) - - -def price_cache_tokens( - model: str, deployment_id: str, tokens: CacheTokenBuckets, completion_tokens: int = 0 -) -> float | None: - try: - selected_model: Final = select_model_name_for_cost_calc( - model=model, - completion_response=None, - custom_pricing=True, - custom_llm_provider="anthropic", - router_model_id=deployment_id, - ) - if selected_model is None: - return None - model_info: Final = litellm.get_model_info(model=selected_model, custom_llm_provider="anthropic") - registry: Final = _PRICE_ENTRY.validate_python(litellm.model_cost) # pyright: ignore[reportUnknownMemberType] # legacy registry is validated at this boundary - price_entry: Final = registry.get(model_info["key"]) - if price_entry is None: - return None - prices: Final = _PRICE_ENTRY.validate_python(price_entry) - if completion_tokens < 0 or not _has_required_prices(prices, tokens, completion_tokens): - return None - usage: Final = Usage( - prompt_tokens=tokens.total_tokens, - completion_tokens=completion_tokens, - total_tokens=tokens.total_tokens + completion_tokens, - prompt_tokens_details=PromptTokensDetailsWrapper( - cached_tokens=tokens.cache_read_input_tokens, - cache_creation_tokens=tokens.cache_creation_5m_input_tokens + tokens.cache_creation_1h_input_tokens, - cache_creation_token_details=CacheCreationTokenDetails( - ephemeral_5m_input_tokens=tokens.cache_creation_5m_input_tokens, - ephemeral_1h_input_tokens=tokens.cache_creation_1h_input_tokens, - ), - ), - ) - logging_obj: Final = Logging( - model=model, - messages=[], - stream=False, - call_type="completion", - start_time=datetime.now(timezone.utc), - litellm_call_id="prompt-cache-prediction", - function_id="prompt-cache-prediction", - ) - completion_cost( - completion_response=ModelResponse(model=model, usage=usage), - model=model, - custom_llm_provider="anthropic", - custom_pricing=True, - router_model_id=deployment_id, - litellm_logging_obj=logging_obj, - ) - breakdown: Final = logging_obj.cost_breakdown - input_cost: Final = breakdown.get("input_cost") if breakdown is not None else None - output_cost: Final = breakdown.get("output_cost") if breakdown is not None else None - if input_cost is None or output_cost is None: - return None - cost: Final = input_cost + output_cost - return cost if _valid_price(cost) else None - except Exception: # noqa: BLE001 # the shared pricing owners raise plain Exception for unpriceable models - return None +__all__ = ("price_cache_tokens",) diff --git a/tests/unit/proxy/common_utils/test_prompt_cache_pricing.py b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_pricing.py similarity index 97% rename from tests/unit/proxy/common_utils/test_prompt_cache_pricing.py rename to tests/unit/llms/anthropic/test_anthropic_prompt_cache_pricing.py index 01b18c1ed71..4f424224012 100644 --- a/tests/unit/proxy/common_utils/test_prompt_cache_pricing.py +++ b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_pricing.py @@ -3,7 +3,7 @@ from typing import Final import pytest import litellm -from litellm.proxy.common_utils.prompt_cache_pricing import price_cache_tokens +from litellm.llms.anthropic.prompt_cache_pricing import price_cache_tokens from litellm.types.management_endpoints.prompt_cache_prediction import CacheTokenBuckets From 4e44b382fd3a2e42d56b67d5092d023a82d1899c Mon Sep 17 00:00:00 2001 From: Marcus Greer Date: Sat, 10 Oct 2026 13:32:54 -0700 Subject: [PATCH 03/40] fix(terraform): stop sending blocked on litellm_user updates (#45410) * fix(terraform): stop sending blocked on litellm_user updates /user/update writes blocked into the user table upsert, but blocked only exists as a column on keys, so every litellm_user update failed with a 400 from Prisma. Updates no longer send it. Create sends it only when set to true, since /user/new applies it to the auto-created key blocked only applies at creation; litellm_key_block covers changes after that * fix(terraform): always send blocked on litellm_user create /user/new fills an omitted blocked from default_internal_user_params, so leaving it out on create let a proxy-side blocked=true default block the auto-created key. Create sends the configured value again, including false, while updates still never send it * chore(terraform): drop comment on litellm_user create blocked * fix(terraform): keep litellm_user blocked undeprecated Create sends blocked again, so it still controls the auto-created key. Whether to deprecate it in favor of litellm_key_block is left to the provider maintainers; the docs describe it as creation-only instead --- terraform/provider/CHANGELOG.md | 1 + terraform/provider/docs/resources/user.md | 2 +- terraform/provider/litellm/resource_user.go | 7 +- .../provider/litellm/resource_user_test.go | 78 +++++++++++++++++++ 4 files changed, 83 insertions(+), 5 deletions(-) diff --git a/terraform/provider/CHANGELOG.md b/terraform/provider/CHANGELOG.md index d8c53e1432c..b948db2b231 100644 --- a/terraform/provider/CHANGELOG.md +++ b/terraform/provider/CHANGELOG.md @@ -49,6 +49,7 @@ longer signal it. ### Fixed +- **user**: `litellm_user` updates no longer send `blocked`, which `/user/update` writes into the user table where no such column exists, so every update failed with a 400 (`Could not find field at upsertOneLiteLLM_UserTable.create.blocked`). Create still sends `blocked`, including `false`, so a `default_internal_user_params.blocked` on the proxy cannot block the auto-created key. Changing `blocked` after creation has no effect, since the proxy has no user-level block; use `litellm_key_block` instead - **model**: `litellm_model` refresh now reads the `{"data": [...]}` envelope `/model/info` returns, so `model_info` fields changed outside Terraform show up as drift instead of silently keeping the previous state - **key**: An update that changes `team_id` and fails because the key was already cascade-deleted along with its previous team now recovers by recreating the key under the new team, instead of aborting the apply. The key's absence is confirmed against the proxy first, so an unrelated failure still errors out, and a `team_id` change between two teams that both still exist stays a plain in-place update - **credential**: create now reports a `credential_name` collision as a clear error naming the `terraform import` command that adopts the existing credential, instead of surfacing the proxy's raw 500 with a Prisma `Unique constraint failed` message. New `adopt_existing` argument (default `false`) opts into taking the existing credential over during create, which makes `apply` idempotent again once state loses track of a credential that still exists on the proxy. Requires a proxy that answers 409 on the collision; older proxies are still detected by their 500 message diff --git a/terraform/provider/docs/resources/user.md b/terraform/provider/docs/resources/user.md index 9b537292d54..19fa8b68f14 100644 --- a/terraform/provider/docs/resources/user.md +++ b/terraform/provider/docs/resources/user.md @@ -50,7 +50,7 @@ resource "litellm_user" "alice" { - `permissions` (Optional) - Map of permission values for the user - `model_max_budget` (Optional) - JSON string of per-model budget config, e.g. `jsonencode({"gpt-4o" = {max_budget = 10.0}})` - `guardrails` (Optional) - List of guardrails applied to the user's requests -- `blocked` (Optional, Default `false`) - Whether the user is blocked from making requests +- `blocked` (Optional, Default `false`) - Whether to block the key auto-created with the user. Only sent on creation, so changing it afterwards has no effect. The proxy has no user-level block; use `litellm_key_block` to block keys ## Attribute Reference diff --git a/terraform/provider/litellm/resource_user.go b/terraform/provider/litellm/resource_user.go index c1aa9d7e9fa..76420d1f9bf 100644 --- a/terraform/provider/litellm/resource_user.go +++ b/terraform/provider/litellm/resource_user.go @@ -164,7 +164,7 @@ func resourceLiteLLMUser() *schema.Resource { Type: schema.TypeBool, Optional: true, Default: false, - Description: "Whether the user is blocked from making requests", + Description: "Whether to block the key auto-created with the user. Only sent on creation; use litellm_key_block to change it afterwards", }, "key": { Type: schema.TypeString, @@ -195,6 +195,7 @@ func resourceLiteLLMUserCreate(d *schema.ResourceData, m interface{}) error { } userData["auto_create_key"] = d.Get("auto_create_key").(bool) userData["send_invite_email"] = d.Get("send_invite_email").(bool) + userData["blocked"] = d.Get("blocked").(bool) log.Printf("[DEBUG] Create user request payload: %+v", userData) @@ -337,9 +338,7 @@ func resourceLiteLLMUserDelete(d *schema.ResourceData, m interface{}) error { } func buildUserData(d *schema.ResourceData) map[string]interface{} { - userData := map[string]interface{}{ - "blocked": d.Get("blocked").(bool), - } + userData := map[string]interface{}{} for _, key := range []string{ "user_email", "user_alias", "user_role", "teams", "models", "max_budget", diff --git a/terraform/provider/litellm/resource_user_test.go b/terraform/provider/litellm/resource_user_test.go index c254c0c5929..046cef91954 100644 --- a/terraform/provider/litellm/resource_user_test.go +++ b/terraform/provider/litellm/resource_user_test.go @@ -239,3 +239,81 @@ func TestResourceUserDelete_SendsUserIDs(t *testing.T) { t.Fatalf("expected ID to be cleared after delete, got %q", d.Id()) } } + +func TestResourceUserUpdate_NeverSendsBlocked(t *testing.T) { + for name, config := range map[string]map[string]interface{}{ + "unset": {"user_role": "internal_user"}, + "true": {"user_role": "internal_user", "blocked": true}, + "false": {"user_role": "internal_user", "blocked": false}, + } { + t.Run(name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/user/update": + var payload map[string]interface{} + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Fatalf("failed to decode update payload: %v", err) + } + if _, ok := payload["blocked"]; ok { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(`{"error": "Could not find field at upsertOneLiteLLM_UserTable.create.blocked"}`)) + return + } + w.Write([]byte(`{"user_id": "u-7"}`)) + case "/user/info": + w.Write(userInfoBody("u-7", map[string]interface{}{"user_role": "internal_user"})) + default: + t.Errorf("unexpected request to %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + defer srv.Close() + + d := schema.TestResourceDataRaw(t, resourceLiteLLMUser().Schema, config) + d.SetId("u-7") + + if err := resourceLiteLLMUserUpdate(d, NewClient(srv.URL, "test-key", true)); err != nil { + t.Fatalf("update failed: %v", err) + } + }) + } +} + +func TestResourceUserCreate_AlwaysSendsBlocked(t *testing.T) { + for name, tc := range map[string]struct { + config map[string]interface{} + wantBlocked interface{} + }{ + "unset": {config: map[string]interface{}{}, wantBlocked: false}, + "false": {config: map[string]interface{}{"blocked": false}, wantBlocked: false}, + "true": {config: map[string]interface{}{"blocked": true}, wantBlocked: true}, + } { + t.Run(name, func(t *testing.T) { + var createPayload map[string]interface{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/user/new": + if err := json.NewDecoder(r.Body).Decode(&createPayload); err != nil { + t.Fatalf("failed to decode create payload: %v", err) + } + w.Write([]byte(`{"user_id": "u-1", "key": "sk-generated"}`)) + case "/user/info": + w.Write(userInfoBody("u-1", map[string]interface{}{"user_role": "internal_user"})) + default: + t.Errorf("unexpected request to %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + defer srv.Close() + + d := schema.TestResourceDataRaw(t, resourceLiteLLMUser().Schema, tc.config) + + if err := resourceLiteLLMUserCreate(d, NewClient(srv.URL, "test-key", true)); err != nil { + t.Fatalf("create failed: %v", err) + } + if got := createPayload["blocked"]; got != tc.wantBlocked { + t.Errorf("expected blocked %v in create payload, got %v", tc.wantBlocked, got) + } + }) + } +} From 3a0803aaa1c4329e3687389ae2dc0b2b53eeffbc Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 10 Oct 2026 13:33:03 -0700 Subject: [PATCH 04/40] fix(ui): store SSO exchange token via storeLoginToken to keep Secure flag (#45840) Co-authored-by: Nabil Ameena Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../src/components/networking.test.ts | 34 +++++++++++++++++++ .../src/components/networking.tsx | 2 +- .../src/utils/cookieUtils.test.ts | 23 +++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/networking.test.ts b/ui/litellm-dashboard/src/components/networking.test.ts index e60c0356751..1aa003750d7 100644 --- a/ui/litellm-dashboard/src/components/networking.test.ts +++ b/ui/litellm-dashboard/src/components/networking.test.ts @@ -118,6 +118,40 @@ describe("loginCall - storeLoginToken integration", () => { }); }); +describe("exchangeLoginCode - storeLoginToken integration", () => { + const originalFetch = global.fetch; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + global.fetch = originalFetch; + }); + + it("calls storeLoginToken when exchange response includes token", async () => { + global.fetch = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ token: "sso-jwt" }), + }) as unknown as typeof global.fetch; + const { storeLoginToken } = await import("@/utils/cookieUtils"); + const token = await Networking.exchangeLoginCode("some-login-code"); + expect(token).toBe("sso-jwt"); + expect(storeLoginToken).toHaveBeenCalledWith("sso-jwt"); + }); + + it("does not call storeLoginToken when exchange response has no token", async () => { + global.fetch = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({}), + }) as unknown as typeof global.fetch; + const { storeLoginToken } = await import("@/utils/cookieUtils"); + const token = await Networking.exchangeLoginCode("some-login-code"); + expect(token).toBeUndefined(); + expect(storeLoginToken).not.toHaveBeenCalled(); + }); +}); + describe("modelInfoCall", () => { let currentFetch: typeof global.fetch; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 2b38da75411..54b2066c9c5 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -7102,7 +7102,7 @@ export const exchangeLoginCode = async (code: string, workerBaseUrl?: string | n const data = await response.json(); if (data.token) { - document.cookie = `token=${data.token}; path=/; SameSite=Lax`; + storeLoginToken(data.token); } return data.token; }; diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.test.ts b/ui/litellm-dashboard/src/utils/cookieUtils.test.ts index a38b96e74c2..a1bf827560b 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.test.ts @@ -174,6 +174,29 @@ describe("cookieUtils", () => { expect(cookieSpy).toHaveBeenCalledWith(expect.stringContaining("path=/ui")); vi.restoreAllMocks(); }); + + it("should write the cookie with Secure on an https page", () => { + vi.stubGlobal("location", { ...window.location, protocol: "https:" }); + const cookieSpy = vi.spyOn(document, "cookie", "set"); + + storeLoginToken("https-jwt"); + + expect(cookieSpy).toHaveBeenCalledWith(expect.stringMatching(/^token=https-jwt; .*; Secure$/)); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it("should write the cookie without Secure on an http page", () => { + vi.stubGlobal("location", { ...window.location, protocol: "http:" }); + const cookieSpy = vi.spyOn(document, "cookie", "set"); + + storeLoginToken("http-jwt"); + + expect(cookieSpy).toHaveBeenCalledWith(expect.stringMatching(/^token=http-jwt; /)); + expect(cookieSpy).not.toHaveBeenCalledWith(expect.stringContaining("Secure")); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); }); describe("getCookie", () => { From 7383f59ad2719ee0edf7725f5fa50e2f7f326919 Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 13:33:16 -0700 Subject: [PATCH 05/40] refactor(skills): move NewSkillRequest to litellm.models.skills (#45768) * refactor(skills): move NewSkillRequest to litellm.models.skills Part of LIT-9423 * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/litellm_proxy/skills/handler.py | 4 ++-- .../llms/litellm_proxy/skills/transformation.py | 2 +- litellm/models/skills.py | 15 +++++++++++++++ litellm/proxy/_types.py | 16 +++------------- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/litellm/llms/litellm_proxy/skills/handler.py b/litellm/llms/litellm_proxy/skills/handler.py index a86301d9a79..0dbe47791fb 100644 --- a/litellm/llms/litellm_proxy/skills/handler.py +++ b/litellm/llms/litellm_proxy/skills/handler.py @@ -15,8 +15,8 @@ from litellm.llms.litellm_proxy.skills.constants import ( LITELLM_SKILL_ID_PREFIX, MAX_SKILLS_PER_SEARCH, ) -from litellm.models.skills import LiteLLM_SkillsTable -from litellm.proxy._types import NewSkillRequest, UserAPIKeyAuth +from litellm.models.skills import LiteLLM_SkillsTable, NewSkillRequest +from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.resource_ownership import ( get_primary_resource_owner_scope, get_resource_owner_scopes, diff --git a/litellm/llms/litellm_proxy/skills/transformation.py b/litellm/llms/litellm_proxy/skills/transformation.py index 834b6e65e02..4cbd9f50301 100644 --- a/litellm/llms/litellm_proxy/skills/transformation.py +++ b/litellm/llms/litellm_proxy/skills/transformation.py @@ -137,7 +137,7 @@ class LiteLLMSkillsTransformationHandler: """Async implementation of create_skill.""" # Lazy import to avoid SDK dependency on proxy from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler - from litellm.proxy._types import NewSkillRequest + from litellm.models.skills import NewSkillRequest skill_request: Final = NewSkillRequest( display_title=display_title, diff --git a/litellm/models/skills.py b/litellm/models/skills.py index f56dcfff49a..e03e62fa731 100644 --- a/litellm/models/skills.py +++ b/litellm/models/skills.py @@ -28,3 +28,18 @@ class LiteLLM_SkillsTable(LiteLLMPydanticObjectBase): created_by: str | None = None updated_at: datetime | None = None updated_by: str | None = None + + +class NewSkillRequest(LiteLLMPydanticObjectBase): + """Request to create a new skill in LiteLLM database""" + + display_title: str | None = None + description: str | None = None + instructions: str | None = None + file_content: bytes | None = None # Binary content of skill files (zip) + file_name: str | None = None # Original filename + file_type: str | None = None # MIME type (e.g., "application/zip") + metadata: dict[str, Any] | None = None + authorization_url: str | None = None + token_url: str | None = None + registration_url: str | None = None diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b1a7d3671fc..26d9e0e1b44 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1809,19 +1809,9 @@ class MCPSubmissionsSummary(LiteLLMPydanticObjectBase): ######## Skills API Types ######## -class NewSkillRequest(LiteLLMPydanticObjectBase): - """Request to create a new skill in LiteLLM database""" - - display_title: str | None = None - description: str | None = None - instructions: str | None = None - file_content: bytes | None = None # Binary content of skill files (zip) - file_name: str | None = None # Original filename - file_type: str | None = None # MIME type (e.g., "application/zip") - metadata: dict[str, Any] | None = None - authorization_url: str | None = None - token_url: str | None = None - registration_url: str | None = None +from litellm.models.skills import ( # noqa: E402 # public re-export + NewSkillRequest as NewSkillRequest, # noqa: PLC0414 # public re-export +) class UpdateSkillRequest(LiteLLMPydanticObjectBase): From 5325c4a236610099c5f06d22a65ed57ccdcf33d0 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:33:55 +0000 Subject: [PATCH 06/40] feat(proxy): per-model team member budgets with router fallback (#43887) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../context_management/editors/compact.py | 66 +- litellm/proxy/_types.py | 11 + litellm/proxy/auth/auth_checks.py | 3 + litellm/proxy/auth/fallback_budget.py | 72 +- litellm/proxy/auth/user_api_key_auth.py | 709 +++++- litellm/proxy/common_request_processing.py | 76 +- .../proxy/hooks/model_max_budget_limiter.py | 60 +- litellm/proxy/litellm_pre_call_utils.py | 3 + .../budget_management_endpoints.py | 9 +- .../management_endpoints/common_utils.py | 55 +- .../management_endpoints/team_endpoints.py | 67 +- .../pass_through_endpoints.py | 1 + litellm/proxy/proxy_server.py | 12 +- litellm/router_utils/common_utils.py | 27 + .../context_management/test_compact.py | 132 +- ...st_auth_checks_object_access_and_lookup.py | 41 + tests/unit/proxy/auth/test_fallback_budget.py | 270 ++- .../test_user_api_key_auth_request_flow.py | 2135 ++++++++++++++++- .../test_upsert_budget_membership.py | 167 +- .../hooks/test_model_max_budget_limiter.py | 123 +- .../management_v1/test_teams.py | 8 +- .../test_budget_endpoints.py | 41 + .../management_endpoints/test_common_utils.py | 136 +- .../test_team_endpoints.py | 362 ++- .../test_management_helpers_utils.py | 9 +- .../test_pass_through_endpoints.py | 9 +- .../proxy/test_common_request_processing.py | 153 +- .../unit/proxy/test_litellm_pre_call_utils.py | 5 + .../test_router_utils_common_utils.py | 20 + .../src/components/Teams.integration.test.tsx | 83 + ui/litellm-dashboard/src/components/Teams.tsx | 14 + .../key_team_helpers/ModelMaxBudgetEditor.tsx | 12 +- .../src/components/networking.tsx | 5 + .../team/EditMembership.integration.test.tsx | 129 + .../src/components/team/EditMembership.tsx | 18 +- .../team/TeamInfo.integration.test.tsx | 94 + .../src/components/team/TeamInfo.tsx | 58 +- .../components/team/TeamMemberTab.test.tsx | 21 + .../src/components/team/TeamMemberTab.tsx | 16 +- .../src/components/team/memberFormValues.ts | 27 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 25 + 41 files changed, 5039 insertions(+), 245 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/Teams.integration.test.tsx diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/compact.py b/litellm/llms/anthropic/pass_through/context_management/editors/compact.py index f743320b702..85a20ea4435 100644 --- a/litellm/llms/anthropic/pass_through/context_management/editors/compact.py +++ b/litellm/llms/anthropic/pass_through/context_management/editors/compact.py @@ -84,6 +84,7 @@ _PROPAGATED_METADATA_KEYS: Final = ( "user_api_key_model_max_budget", "user_api_key_team_model_max_budget", "user_api_key_user_model_max_budget", + "user_api_key_team_member_model_max_budget", "user_api_key_end_user_model_max_budget", "litellm_call_id", "litellm_parent_otel_span", @@ -125,6 +126,23 @@ class _SummaryAcompletion(Protocol): ) -> "Awaitable[ModelResponse | CustomStreamWrapper]": ... +class _SummaryModelBudgetLimiter(Protocol): + async def is_team_member_within_model_budget( + self, + user_id: str, + team_id: str, + team_member_model_max_budget: Mapping[str, object], + model: str, + ) -> bool: ... + + async def is_end_user_within_model_budget( + self, + end_user_id: str, + end_user_model_max_budget: Mapping[str, object], + model: str, + ) -> bool: ... + + class _CreateRateLimitDescriptors(Protocol): def __call__( self, @@ -392,6 +410,42 @@ async def _check_summary_model_access( return True +async def _check_summary_team_member_model_budget( + user_api_key_auth: "UserAPIKeyAuth", + model_max_budget_limiter: _SummaryModelBudgetLimiter, + summary_model: str, +) -> bool: + team_member_model_max_budget: Final = user_api_key_auth.team_member_model_max_budget + user_id: Final = user_api_key_auth.user_id + team_id: Final = user_api_key_auth.team_id + if ( + not isinstance(team_member_model_max_budget, Mapping) + or not team_member_model_max_budget + or not isinstance(user_id, str) + or not isinstance(team_id, str) + ): + return True + + try: + await model_max_budget_limiter.is_team_member_within_model_budget( + user_id=user_id, + team_id=team_id, + team_member_model_max_budget=team_member_model_max_budget, + model=summary_model, + ) + except litellm.BudgetExceededError: + return False + except Exception as e: # noqa: BLE001 # unexpected budget-check errors must deny compaction + verbose_logger.warning( + "compact_20260112: unexpected error during team member model-budget " + "check for summary_model=%s; denying: %s", + summary_model, + e, + ) + return False + return True + + async def _check_summary_model_budget( user_api_key_auth: Optional["UserAPIKeyAuth"], summary_model: str, @@ -407,9 +461,9 @@ async def _check_summary_model_budget( per-model budget is configured. Every scope is checked because the summary's spend is charged to every - scope: this file propagates the key, team, user and end-user budgets into the - subrequest's metadata, so skipping one of them would let compaction - increment a counter it can never be refused by. + scope: this file propagates the key, team, user, team-member and end-user + budgets into the subrequest's metadata, so skipping one of them would let + compaction increment a counter it can never be refused by. """ if user_api_key_auth is None: return True @@ -497,7 +551,11 @@ async def _check_summary_model_budget( ) return False - return True + return await _check_summary_team_member_model_budget( + user_api_key_auth=user_api_key_auth, + model_max_budget_limiter=model_max_budget_limiter, + summary_model=summary_model, + ) def _without_parallel_request_gauges( diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 26d9e0e1b44..7ac8b1f8538 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2106,6 +2106,7 @@ class NewTeamRequest(TeamBase): team_member_tpm_limit: int | None = None # allow user to set TPM limit for all team members team_member_key_duration: str | None = None # e.g. "1d", "1w", "1m" team_member_budget_duration: str | None = None # e.g. "30d", "1mo" + team_member_model_max_budget: GenericBudgetConfigType | None = None allowed_vector_store_indexes: list[AllowedVectorStoreIndexItem] | None = None enforced_batch_output_expires_after: dict | None = None enforced_file_expires_after: dict | None = None @@ -2165,6 +2166,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase): require_trace_id: bool | None = None team_member_budget: float | None = None team_member_budget_duration: str | None = None + team_member_model_max_budget: GenericBudgetConfigType | None = None team_member_rpm_limit: int | None = None team_member_tpm_limit: int | None = None team_member_key_duration: str | None = None @@ -3363,6 +3365,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob # and validating it here would make one malformed row fail auth outright. # resolve_model_budget validates the single entry a request actually needs. user_model_max_budget: Mapping[str, object] | None = None + team_member_model_max_budget: Mapping[str, object] | None = Field(default=None, exclude=True) request_route: str | None = None is_session_token: bool = False # Server-only marker set exclusively by the MCP gateway admission path @@ -4651,6 +4654,13 @@ class TeamMemberUpdateRequest(TeamMemberDeleteRequest): default=None, description="UTC expiry for temp_budget_increase", ) + model_max_budget: GenericBudgetConfigType | None = Field( + default=None, + description=( + "Per-model spend caps for this team member, each with its own budget_duration. " + "Overrides the team's default per-model member budget. Pass an empty dict to fall back to the team default." + ), + ) @model_validator(mode="after") def validate_temp_budget(self) -> "TeamMemberUpdateRequest": @@ -4669,6 +4679,7 @@ class TeamMemberUpdateResponse(MemberUpdateResponse): allowed_models: list[str] | None = None temp_budget_increase: float | None = None temp_budget_expiry: datetime | None = None + model_max_budget: GenericBudgetConfigType | None = None class TeamModelAddRequest(LiteLLMBaseModel): diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 7c62e4f4147..d687b2ad7a0 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -1739,6 +1739,7 @@ async def get_team_member_default_budget( budget_id: str, prisma_client: PrismaClient | None, user_api_key_cache: UserApiKeyCache, + raise_on_lookup_error: bool = False, ) -> LiteLLM_BudgetTable | None: """ Fetches the team-level default per-member budget referenced by team.metadata["team_member_budget_id"]. @@ -1773,6 +1774,8 @@ async def get_team_member_default_budget( ) except Exception: verbose_proxy_logger.exception("Error fetching team-default member budget %s", budget_id) + if raise_on_lookup_error: + raise return None if budget_record is None: diff --git a/litellm/proxy/auth/fallback_budget.py b/litellm/proxy/auth/fallback_budget.py index 5aa28a08bb8..a126a53821b 100644 --- a/litellm/proxy/auth/fallback_budget.py +++ b/litellm/proxy/auth/fallback_budget.py @@ -11,13 +11,13 @@ attempted, mirroring `fallback_model_access.py`. It deliberately leaves the prim a zero-cost model is never blocked by budget, and only the paid fallback is refused. On by default; set `general_settings.enforce_fallback_budget: false` to restore the unguarded behaviour. -Scope: the key's and the user's `max_budget`. Not covered yet, and each needs a read-only evaluation -path before it can be: team, team-member, end-user, org, global and per-model budgets, whose -auth-path functions enforce rather than report (they raise), so reusing them would fire threshold -alerts and take spend reservations for a target that is then skipped; and the key's rolling -`budget_limits` windows, whose accumulated spend lives only in per-window counters -(`spend:key:{token}:window:{budget_duration}`), so enforcing them means more counter reads on the -fallback path rather than reusing state auth already loaded. +Scope: the key's and the user's `max_budget`, plus the team member's per-model caps. Not covered yet, +and each needs a read-only evaluation path before it can be: team and team-member overall, end-user, +org, global and other per-model budgets, whose auth-path functions enforce rather than report (they +raise), so reusing them would fire threshold alerts and take spend reservations for a target that is +then skipped; and the key's rolling `budget_limits` windows, whose accumulated spend lives only in +per-window counters (`spend:key:{token}:window:{budget_duration}`), so enforcing them means more +counter reads on the fallback path rather than reusing state auth already loaded. Two known limitations of that narrow scope, both shared with `fallback_model_access.py`: @@ -33,10 +33,11 @@ Two known limitations of that narrow scope, both shared with `fallback_model_acc from collections.abc import Callable, Mapping from dataclasses import dataclass -from typing import Final +from typing import Final, Protocol from pydantic import ValidationError +import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module exports @@ -55,6 +56,16 @@ class _FallbackBudgetSettings(LiteLLMBaseModel): enforce_fallback_budget: bool = True +class _TeamMemberModelBudgetLimiter(Protocol): + async def is_team_member_within_model_budget( + self, + user_id: str, + team_id: str, + team_member_model_max_budget: Mapping[str, object], + model: str, + ) -> bool: ... + + def _token_in_metadata(metadata: object) -> UserAPIKeyAuth | None: try: return _RequestMetadata.model_validate(metadata).user_api_key_auth @@ -103,7 +114,13 @@ async def _counter_spend(counter_key: str, fallback_spend: float, max_budget: fl ) -async def is_token_within_budget_for_model(*, model: str, valid_token: UserAPIKeyAuth, llm_router: Router) -> bool: +async def is_token_within_budget_for_model( + *, + model: str, + valid_token: UserAPIKeyAuth, + llm_router: Router, + team_member_model_budget_limiter: _TeamMemberModelBudgetLimiter | None = None, +) -> bool: """ True when the key and the user behind it can still pay for `model`. @@ -139,6 +156,23 @@ async def is_token_within_budget_for_model(*, model: str, valid_token: UserAPIKe if user_spend >= user_budget: return False + team_member_model_max_budget: Final = valid_token.team_member_model_max_budget + if ( + team_member_model_budget_limiter is not None + and team_member_model_max_budget + and isinstance(valid_token.user_id, str) + and isinstance(valid_token.team_id, str) + ): + try: + await team_member_model_budget_limiter.is_team_member_within_model_budget( + user_id=valid_token.user_id, + team_id=valid_token.team_id, + team_member_model_max_budget=team_member_model_max_budget, + model=model, + ) + except litellm.BudgetExceededError: + return False + return True @@ -151,6 +185,7 @@ class RouterFallbackBudgetCheck: """ is_enforced: Callable[[], bool] + team_member_model_budget_limiter: _TeamMemberModelBudgetLimiter | None = None async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool: if not self.is_enforced(): @@ -158,11 +193,20 @@ class RouterFallbackBudgetCheck: valid_token: Final = _user_api_key_auth_from_request(request_kwargs) if valid_token is None: return True - try: - return await is_token_within_budget_for_model(model=model, valid_token=valid_token, llm_router=llm_router) - except Exception as e: # noqa: BLE001 # fail closed: a spend lookup failure must not bill the caller - verbose_proxy_logger.warning("Skipping fallback to model=%s: budget lookup failed: %s", model, e) - return False + budget_models: Final = tuple(dict.fromkeys((model, llm_router.routable_model_group(model)))) + for budget_model in budget_models: + try: + if not await is_token_within_budget_for_model( + model=budget_model, + valid_token=valid_token, + llm_router=llm_router, + team_member_model_budget_limiter=self.team_member_model_budget_limiter, + ): + return False + except Exception as e: # noqa: BLE001 # fail closed: a spend lookup failure must not bill the caller + verbose_proxy_logger.warning("Skipping fallback to model=%s: budget lookup failed: %s", model, e) + return False + return True router_fallback_budget_check: Final = RouterFallbackBudgetCheck(is_enforced=_enforced_by_general_settings) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index d6a51bc2fc9..2231706b282 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -11,12 +11,11 @@ import asyncio import fnmatch import re import secrets -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import datetime, timezone from typing import Final, NamedTuple, Protocol, Union, cast import fastapi -import orjson from fastapi import HTTPException, Request, WebSocket, status from fastapi.security.api_key import APIKeyHeader from starlette.exceptions import WebSocketException @@ -51,13 +50,18 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo _get_user_role, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _is_model_cost_zero, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _is_user_proxy_admin, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _model_access_group_max_budget_check, # pyright: ignore[reportPrivateUsage] # reuse common-check group budget enforcement _team_member_max_budget_alert_check, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _unpriced_models_in_request, # pyright: ignore[reportPrivateUsage] # reuse common-check pricing enforcement _virtual_key_max_budget_alert_check, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _virtual_key_max_budget_check, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _virtual_key_soft_budget_check, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export cache_key_object, + can_customer_access_model, can_key_call_model, can_object_call_model, + can_project_access_model, + can_team_access_model, check_end_user_budget, common_checks, delete_cache_key_object, @@ -67,6 +71,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo get_object_permission, get_org_object_for_request, get_project_object, + get_team_member_default_budget, get_team_membership, get_team_object, get_user_object, @@ -76,9 +81,12 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo is_user_proxy_admin, is_valid_fallback_model, jwt_key_mapping_cache_key, + key_access_group_grants_model, key_model_aliases_for_auth_check, resolve_and_validate_end_user_id, resolve_default_end_user_budget, + route_skips_budget_checks, + stamp_matched_model_access_groups, team_member_max_budget_alert_check, virtual_key_max_budget_alert_check, virtual_key_max_budget_check, @@ -169,7 +177,13 @@ from litellm.proxy.utils import ( ) from litellm.repositories.table_repositories import JWTKeyMappingRepository, TeamMembershipRepository from litellm.repositories.verification_token_repository import VerificationTokenRepository -from litellm.router_utils.common_utils import resolve_model_group_alias +from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model +from litellm.router_utils.fallback_event_handlers import ( + PROVIDER_SCOPED_RESOURCE_KEYS, + fallback_lookup_groups, + fallbacks_disabled_for_request, + get_fallback_model_group_for_lookup_groups, +) from litellm.secret_managers.main import get_secret_bool from litellm.types.services import ServiceTypes @@ -332,6 +346,18 @@ def _get_model_names_for_budget_checks( return model +def _budget_fallback_target_model(target: object) -> str | None: + if isinstance(target, str): + return target + if not isinstance(target, Mapping): + return None + target_mapping: Final = cast(Mapping[str, object], target) # cast-ok: isinstance cannot type the container's items + if set(target_mapping) != {"model"}: + return None + model: Final = target_mapping.get("model") + return model if isinstance(model, str) else None + + class _KeyModelBudgetLimiter(Protocol): async def is_key_within_model_budget(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool: ... @@ -354,6 +380,34 @@ class _TeamModelBudgetLimiter(Protocol): ) -> bool: ... +class _EveryModelBudgetLimiter(Protocol): + async def is_key_within_model_budget(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool: ... + + async def is_user_within_model_budget( + self, user_id: str, user_model_max_budget: Mapping[str, object], model: str + ) -> bool: ... + + async def is_end_user_within_model_budget( + self, end_user_id: str, end_user_model_max_budget: Mapping[str, object], model: str + ) -> bool: ... + + async def is_team_within_model_budget( + self, + team_id: str, + team_model_max_budget: Mapping[str, object], + key_model_max_budget: Mapping[str, object] | None, + model: str, + ) -> bool: ... + + async def is_team_member_within_model_budget( + self, + user_id: str, + team_id: str, + team_member_model_max_budget: Mapping[str, object], + model: str, + ) -> bool: ... + + class _TokenTeamModels(Protocol): @property def team_models(self) -> list[str]: ... @@ -374,6 +428,30 @@ def _token_team_models(valid_token: _TokenTeamModels) -> list[str]: return valid_token.team_models +class _TokenModelBudgetFields(Protocol): + @property + def model_max_budget(self) -> Mapping[str, object] | None: ... + + @property + def end_user_model_max_budget(self) -> Mapping[str, object] | None: ... + + @property + def team_model_aliases(self) -> dict[str, str] | None: ... # mutable-ok: mirrors _can_object_call_model's contract + + +def _token_model_budget_fields(valid_token: _TokenModelBudgetFields) -> _TokenModelBudgetFields: + return valid_token + + +class _RouterFallbacks(Protocol): + @property + def fallbacks(self) -> Sequence[object] | None: ... + + +def _router_fallbacks(router: _RouterFallbacks) -> Sequence[object] | None: + return router.fallbacks + + async def _read_user_model_max_budget( user_id: str | None, prisma_client: PrismaClient | None, @@ -443,13 +521,455 @@ async def _check_team_model_budget( ) +async def _is_within_every_model_budget( + valid_token: UserAPIKeyAuth, + model_max_budget_limiter: _EveryModelBudgetLimiter, + model: str, +) -> bool: + budget_fields: Final = _token_model_budget_fields(valid_token) + key_model_max_budget: Final = budget_fields.model_max_budget + user_model_max_budget: Final = valid_token.user_model_max_budget + end_user_model_max_budget: Final = budget_fields.end_user_model_max_budget + team_model_max_budget: Final = valid_token.team_model_max_budget + team_member_model_max_budget: Final = valid_token.team_member_model_max_budget + try: + if valid_token.token is not None and isinstance(key_model_max_budget, Mapping) and key_model_max_budget: + await model_max_budget_limiter.is_key_within_model_budget( + user_api_key_dict=valid_token, + model=model, + ) + if valid_token.user_id is not None and isinstance(user_model_max_budget, Mapping) and user_model_max_budget: + await model_max_budget_limiter.is_user_within_model_budget( + user_id=valid_token.user_id, + user_model_max_budget=user_model_max_budget, + model=model, + ) + if ( + valid_token.end_user_id is not None + and isinstance(end_user_model_max_budget, Mapping) + and end_user_model_max_budget + ): + await model_max_budget_limiter.is_end_user_within_model_budget( + end_user_id=valid_token.end_user_id, + end_user_model_max_budget=end_user_model_max_budget, + model=model, + ) + if valid_token.team_id is not None and isinstance(team_model_max_budget, Mapping) and team_model_max_budget: + await model_max_budget_limiter.is_team_within_model_budget( + team_id=valid_token.team_id, + team_model_max_budget=team_model_max_budget, + key_model_max_budget=key_model_max_budget, + model=model, + ) + if ( + valid_token.user_id is not None + and valid_token.team_id is not None + and isinstance(team_member_model_max_budget, Mapping) + and team_member_model_max_budget + ): + await model_max_budget_limiter.is_team_member_within_model_budget( + user_id=valid_token.user_id, + team_id=valid_token.team_id, + team_member_model_max_budget=team_member_model_max_budget, + model=model, + ) + except litellm.BudgetExceededError: + return False + return True + + +async def _is_fallback_model_authorized( + model: str, + valid_token: UserAPIKeyAuth, + llm_model_list: Sequence[object] | None, + llm_router: litellm.Router | None, + prisma_client: PrismaClient | None = None, + team_membership: LiteLLM_TeamMembership | None = None, + project_object: LiteLLM_ProjectTableCachedObj | None = None, + refuse_agent_keys: bool = False, + team_object: LiteLLM_TeamTable | None = None, +) -> bool: + if refuse_agent_keys and valid_token.agent_id is not None: + return False + try: + await can_key_call_model( + model=model, + llm_model_list=llm_model_list, + valid_token=valid_token, + llm_router=llm_router, + prisma_client=prisma_client, + ) + if team_object is not None: + try: + await can_team_access_model( + model=model, + team_object=team_object, + llm_router=llm_router, + team_model_aliases=_token_model_budget_fields(valid_token).team_model_aliases, + key_model_aliases=key_model_aliases_for_auth_check(valid_token), + prisma_client=prisma_client, + ) + except ProxyException as team_denial: + if team_denial.type != ProxyErrorTypes.team_model_access_denied: + raise + if not await key_access_group_grants_model( + model=model, + valid_token=valid_token, + team_object=team_object, + llm_router=llm_router, + key_model_aliases=key_model_aliases_for_auth_check(valid_token), + ): + raise + else: + team_models: Final = _token_team_models(valid_token) + if team_models: + can_object_call_model( + model=model, + llm_router=llm_router, + models=team_models, + team_model_aliases=_token_model_budget_fields(valid_token).team_model_aliases, + team_id=valid_token.team_id, + key_model_aliases=key_model_aliases_for_auth_check(valid_token), + object_type="team", + ) + member_allowed_models: Final = ( + team_membership.litellm_budget_table.allowed_models + if team_membership is not None and team_membership.litellm_budget_table is not None + else None + ) + if member_allowed_models: + can_object_call_model( + model=model, + llm_router=llm_router, + models=member_allowed_models, + team_id=valid_token.team_id, + key_model_aliases=key_model_aliases_for_auth_check(valid_token), + object_type="team", + ) + if project_object is not None and len(project_object.models) > 0: + can_project_access_model( + model=model, + project_object=project_object, + llm_router=llm_router, + key_model_aliases=key_model_aliases_for_auth_check(valid_token), + ) + except ProxyException: + return False + return True + + +async def _is_within_all_fallback_model_budgets( + valid_token: UserAPIKeyAuth, + model_max_budget_limiter: _EveryModelBudgetLimiter, + models: Sequence[str], +) -> bool: + for model in models: + if not await _is_within_every_model_budget( + valid_token=valid_token, + model_max_budget_limiter=model_max_budget_limiter, + model=model, + ): + return False + return True + + +async def _check_team_member_model_budget_with_fallback( + valid_token: UserAPIKeyAuth, + model_max_budget_limiter: _EveryModelBudgetLimiter, + models: Sequence[str], + request_data: dict[str, object], # mutable-ok: fallback rewrites the request body + request: Request, + llm_model_list: Sequence[object] | None, + llm_router: litellm.Router | None, + project_object: LiteLLM_ProjectTableCachedObj | None = None, + team_object: LiteLLM_TeamTable | None = None, + end_user_object: LiteLLM_EndUserTable | None = None, + route: str = "", + prisma_client: PrismaClient | None = None, + user_api_key_cache: UserApiKeyCache | None = None, + proxy_logging_obj: ProxyLogging | None = None, +) -> None: + team_member_model_max_budget: Final = valid_token.team_member_model_max_budget + if ( + valid_token.user_id is None + or valid_token.team_id is None + or not isinstance(team_member_model_max_budget, Mapping) + or not team_member_model_max_budget + ): + return + + user_id: Final = valid_token.user_id + team_id: Final = valid_token.team_id + for model_name in models: + await _check_team_member_model_budget_for_model( + user_id=user_id, + team_id=team_id, + team_member_model_max_budget=team_member_model_max_budget, + model_name=model_name, + valid_token=valid_token, + model_max_budget_limiter=model_max_budget_limiter, + request_data=request_data, + request=request, + llm_model_list=llm_model_list, + llm_router=llm_router, + project_object=project_object, + team_object=team_object, + end_user_object=end_user_object, + route=route, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + +async def _check_team_member_model_budget_for_model( + user_id: str, + team_id: str, + team_member_model_max_budget: Mapping[str, object], + model_name: str, + valid_token: UserAPIKeyAuth, + model_max_budget_limiter: _EveryModelBudgetLimiter, + request_data: dict[str, object], # mutable-ok: fallback rewrites the request body + request: Request, + llm_model_list: Sequence[object] | None, + llm_router: litellm.Router | None, + project_object: LiteLLM_ProjectTableCachedObj | None = None, + team_object: LiteLLM_TeamTable | None = None, + end_user_object: LiteLLM_EndUserTable | None = None, + route: str = "", + prisma_client: PrismaClient | None = None, + user_api_key_cache: UserApiKeyCache | None = None, + proxy_logging_obj: ProxyLogging | None = None, +) -> None: + try: + await model_max_budget_limiter.is_team_member_within_model_budget( + user_id=user_id, + team_id=team_id, + team_member_model_max_budget=team_member_model_max_budget, + model=model_name, + ) + except litellm.BudgetExceededError: + if ( + request_data.get("model") != model_name + or llm_router is None + or fallbacks_disabled_for_request(kwargs=request_data) + ): + raise + fallback_succeeded: Final = await _try_team_member_model_budget_fallback( + user_id=user_id, + team_id=team_id, + model_name=model_name, + valid_token=valid_token, + model_max_budget_limiter=model_max_budget_limiter, + request_data=request_data, + request=request, + llm_model_list=llm_model_list, + llm_router=llm_router, + project_object=project_object, + team_object=team_object, + end_user_object=end_user_object, + route=route, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + if not fallback_succeeded: + raise + + +async def _try_team_member_model_budget_fallback( + user_id: str, + team_id: str, + model_name: str, + valid_token: UserAPIKeyAuth, + model_max_budget_limiter: _EveryModelBudgetLimiter, + request_data: dict[str, object], # mutable-ok: fallback rewrites the request body + request: Request, + llm_model_list: Sequence[object] | None, + llm_router: litellm.Router, + project_object: LiteLLM_ProjectTableCachedObj | None = None, + team_object: LiteLLM_TeamTable | None = None, + end_user_object: LiteLLM_EndUserTable | None = None, + route: str = "", + prisma_client: PrismaClient | None = None, + user_api_key_cache: UserApiKeyCache | None = None, + proxy_logging_obj: ProxyLogging | None = None, +) -> bool: + if request.headers.get("content-type", "").lower().startswith("multipart/form-data") or any( + request_data.get(key) for key in PROVIDER_SCOPED_RESOURCE_KEYS + ): + return False + + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import proxy_config + + resolved_prisma_client: Final = prisma_client if prisma_client is not None else proxy_server.prisma_client + resolved_user_api_key_cache: Final = ( + user_api_key_cache if user_api_key_cache is not None else proxy_server.user_api_key_cache + ) + resolved_proxy_logging_obj: Final = ( + proxy_logging_obj if proxy_logging_obj is not None else proxy_server.proxy_logging_obj + ) + + try: + router_settings: Final[dict[str, object] | None] = ( + cast( # cast-ok: proxy settings return an unparameterized dict + dict[str, object] | None, + await proxy_config.get_hierarchical_router_settings( + user_api_key_dict=valid_token, + prisma_client=resolved_prisma_client, + proxy_logging_obj=resolved_proxy_logging_obj, + ), + ) + ) + except Exception: # noqa: BLE001 # Settings lookup failures must not authorize a fallback. + return False + + configured_fallbacks: Final[object] = router_settings.get("fallbacks") if router_settings is not None else None + router_fallbacks: Final[Sequence[object] | None] = ( + cast(Sequence[object], configured_fallbacks) # cast-ok: guarded by the list check below + if isinstance(configured_fallbacks, list) + else _router_fallbacks(llm_router) + ) + if router_fallbacks is None: + return False + + fallback_models, _ = get_fallback_model_group_for_lookup_groups( + fallbacks=list(router_fallbacks), + lookup_groups=fallback_lookup_groups(kwargs=request_data, model_group=model_name), + ) + fallback_targets: Final[Sequence[object]] = fallback_models or () + fallback_candidates: Final[tuple[str, ...]] = tuple( + model for target in fallback_targets if (model := _budget_fallback_target_model(target)) is not None + ) + if not fallback_candidates: + return False + + model_group_aliases: Final[object] = ( + router_settings.get("model_group_alias") if router_settings is not None else None + ) + key_aliases: Final[object] = cast( # cast-ok: auth aliases are dynamically typed at runtime + object, valid_token.aliases + ) + resolved_fallback_candidates: Final[tuple[tuple[str, str], ...]] = tuple( + ( + candidate, + target if (target := resolve_model_group_alias(model_group_aliases, candidate)) is not None else candidate, + ) + for candidate in fallback_candidates + ) + fallback_candidates_with_served_models: Final = ( + ( + fallback_candidate, + fallback_model, + resolve_served_model( + fallback_model, + litellm.model_alias_map, + key_aliases, + model_group_aliases, + ), + ) + for fallback_candidate, fallback_model in resolved_fallback_candidates + ) + + try: + team_membership: Final = await get_team_membership( + user_id=user_id, + team_id=team_id, + prisma_client=resolved_prisma_client, + user_api_key_cache=resolved_user_api_key_cache, + proxy_logging_obj=resolved_proxy_logging_obj, + ) + except Exception: # noqa: BLE001 # membership lookup failures must not authorize a fallback + return False + + for fallback_candidate, fallback_model, served_model in fallback_candidates_with_served_models: + if fallback_model == model_name: + continue + try: + if ( + litellm.block_requests_for_models_without_pricing + and RouteChecks.is_llm_api_route(route=route) + and _unpriced_models_in_request(model=served_model, llm_router=llm_router) + ): + continue + if end_user_object is not None and end_user_object.models: + can_customer_access_model( + model=fallback_model, + end_user_object=end_user_object, + llm_router=llm_router, + valid_token=valid_token, + ) + if not is_model_cost_zero( + model=served_model, llm_router=llm_router + ) and not await _is_within_all_fallback_model_budgets( + valid_token=valid_token, + model_max_budget_limiter=model_max_budget_limiter, + models=tuple( + dict.fromkeys( + ( + fallback_candidate, + fallback_model, + served_model, + llm_router.routable_model_group(served_model), + ) + ) + ), + ): + continue + if not await _is_fallback_model_authorized( + model=fallback_model, + valid_token=valid_token, + llm_model_list=llm_model_list, + llm_router=llm_router, + prisma_client=prisma_client, + team_object=team_object, + team_membership=team_membership, + project_object=project_object, + refuse_agent_keys=True, + ): + continue + valid_token.matched_model_access_groups = [] # rebind-ok: clear prior groups before evaluating this candidate + matched_groups = await stamp_matched_model_access_groups( + model=fallback_model, + valid_token=valid_token, + team_object=team_object, + project_object=project_object, + llm_router=llm_router, + prisma_client=resolved_prisma_client, + user_api_key_cache=resolved_user_api_key_cache, + proxy_logging_obj=resolved_proxy_logging_obj, + team_membership=team_membership, + team_membership_loaded=True, + ) + valid_token.matched_model_access_groups = list(matched_groups) # rebind-ok: spend uses the candidate groups + if matched_groups: + await _model_access_group_max_budget_check( + matched_model_access_groups=list(matched_groups), + prisma_client=resolved_prisma_client, + user_api_key_cache=resolved_user_api_key_cache, + ) + except Exception: # noqa: BLE001 # A candidate check failure must not escape the fail-closed walk. + verbose_proxy_logger.warning( + "Skipping fallback candidate %s after a budget or authorization check error", + fallback_model.replace("\r", "").replace("\n", ""), + exc_info=True, + ) + continue + rewrite_request_model(request_data=request_data, request=request, model=fallback_model) + if "path_params" in request.scope: + request.scope["path_params"]["model"] = fallback_model # rebind-ok: route handlers use the fallback model + return True + return False + + async def _check_key_model_budget_with_fallback( valid_token: UserAPIKeyAuth, model_max_budget_limiter: _KeyModelBudgetLimiter, model_name: str, - request_data: dict, + request_data: dict[str, object], # mutable-ok: fallback rewrites the request body request: Request, - llm_model_list: list | None = None, + llm_model_list: Sequence[object] | None = None, llm_router: litellm.Router | None = None, ) -> None: """ @@ -489,32 +1009,16 @@ async def _check_key_model_budget_with_fallback( ) if fallback_model is None: raise e - try: - await can_key_call_model( - model=fallback_model, - llm_model_list=llm_model_list, - valid_token=valid_token, - llm_router=llm_router, - ) - if valid_token.team_models: - can_object_call_model( - model=fallback_model, - llm_router=llm_router, - models=valid_token.team_models, - team_model_aliases=valid_token.team_model_aliases, - team_id=valid_token.team_id, - key_model_aliases=key_model_aliases_for_auth_check(valid_token), - object_type="team", - ) - except ProxyException: - raise e - request_data["model"] = fallback_model - safe_set_request_parsed_body(request=request, parsed_body=request_data) - request._json = request_data # pyright: ignore[reportPrivateUsage] # Starlette JSON cache - request._body = orjson.dumps(request_data) # pyright: ignore[reportPrivateUsage] # Starlette body cache - path_params: Final = request.scope.get("path_params") - if isinstance(path_params, dict) and "model" in path_params: - path_params["model"] = fallback_model + if not await _is_fallback_model_authorized( + model=fallback_model, + valid_token=valid_token, + llm_model_list=llm_model_list, + llm_router=llm_router, + ): + raise + rewrite_request_model(request_data=request_data, request=request, model=fallback_model) + if "path_params" in request.scope: + request.scope["path_params"]["model"] = fallback_model # rebind-ok: route handlers use the fallback model def _get_bearer_token_or_received_api_key(api_key: str) -> str: @@ -2543,7 +3047,9 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e model_name=model_name, request_data=request_data, request=request, - llm_model_list=llm_model_list, + llm_model_list=cast( # cast-ok: router model lists are deployment sequences + Sequence[object] | None, llm_model_list + ), llm_router=llm_router, ) @@ -2836,6 +3342,79 @@ def is_no_auth_dev_mode(master_key: str | None, general_settings: Mapping[str, o ) +async def _load_team_member_default_model_budget( + user_api_key_auth_obj: UserAPIKeyAuth, + team_object: LiteLLM_TeamTable | None, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, + skip_budget_checks: bool, + proxy_logging_obj: ProxyLogging | None, +) -> None: + team_metadata_value: Final[object] = getattr(team_object, "metadata", None) if team_object is not None else None + team_metadata: Final[Mapping[str, object] | None] = ( + cast( # cast-ok: team metadata is a string-keyed JSON object + Mapping[str, object], team_metadata_value + ) + if isinstance(team_metadata_value, Mapping) + else None + ) + team_member_budget_id: Final[object | None] = ( + team_metadata.get("team_member_budget_id") if isinstance(team_metadata, Mapping) else None + ) + if ( + not isinstance(user_api_key_auth_obj.user_id, str) + or not isinstance(user_api_key_auth_obj.team_id, str) + or prisma_client is None + ): + return + + try: + membership: Final = await get_team_membership( + user_id=user_api_key_auth_obj.user_id, + team_id=user_api_key_auth_obj.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + member_budget_row: Final = membership.litellm_budget_table if membership is not None else None + member_budget_value: Final[object] = ( + getattr(member_budget_row, "model_max_budget", None) if member_budget_row is not None else None + ) + if ( + membership is not None + and membership.budget_id is not None + and membership.budget_id != team_member_budget_id + and isinstance(member_budget_value, Mapping) + and member_budget_value + ): + user_api_key_auth_obj.team_member_model_max_budget = ( # rebind-ok: auth context carries member budget downstream + member_budget_value + ) + return + if not isinstance(team_member_budget_id, str): + return + team_member_default_budget: Final = await get_team_member_default_budget( + budget_id=team_member_budget_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + raise_on_lookup_error=True, + ) + except Exception: + if not skip_budget_checks: + raise + verbose_proxy_logger.warning("Skipping team-member model budget lookup after a lookup error", exc_info=True) + return + + default_budget_value: Final[object] = ( + getattr(team_member_default_budget, "model_max_budget", None) + if team_member_default_budget is not None + else None + ) + user_api_key_auth_obj.team_member_model_max_budget = ( # rebind-ok: auth context carries default budget downstream + default_budget_value if isinstance(default_budget_value, Mapping) else None + ) + + @tracer.wrap() async def run_centralized_common_checks( user_api_key_auth_obj: UserAPIKeyAuth, @@ -2864,6 +3443,7 @@ async def run_centralized_common_checks( from litellm.proxy.proxy_server import ( general_settings, litellm_proxy_admin_name, + llm_model_list, # pyright: ignore[reportUnknownVariableType] # proxy global has untyped deployment entries llm_router, master_key, model_max_budget_limiter, @@ -3124,6 +3704,17 @@ async def run_centralized_common_checks( llm_router=llm_router, team_id=user_api_key_auth_obj.team_id, ) + typed_team_object: Final[LiteLLM_TeamTable | None] = ( + team_object if isinstance(team_object, LiteLLM_TeamTable) else None + ) + await _load_team_member_default_model_budget( + user_api_key_auth_obj=user_api_key_auth_obj, + team_object=typed_team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + skip_budget_checks=skip_budget_checks or route_skips_budget_checks(route=route), + proxy_logging_obj=proxy_logging_obj, + ) # Pin the metadata variable name (litellm_metadata vs metadata) before # any tag merge runs. Without this, header tags from @@ -3164,19 +3755,42 @@ async def run_centralized_common_checks( project_object=project_object, ) if not skip_budget_checks: + models: Final = _get_model_names_for_budget_checks( + model=_get_model_from_request_context( + request_data=request_data, + route=route, + request=request, + llm_router=llm_router, + team_id=user_api_key_auth_obj.team_id, + ) + ) await _check_team_model_budget( valid_token=user_api_key_auth_obj, model_max_budget_limiter=model_max_budget_limiter, - models=_get_model_names_for_budget_checks( - model=_get_model_from_request_context( - request_data=request_data, - route=route, - request=request, - llm_router=llm_router, - team_id=user_api_key_auth_obj.team_id, - ) - ), + models=models, ) + typed_project_object: Final[LiteLLM_ProjectTableCachedObj | None] = ( + project_object if isinstance(project_object, LiteLLM_ProjectTableCachedObj) else None + ) + if not route_skips_budget_checks(route=route): + await _check_team_member_model_budget_with_fallback( + valid_token=user_api_key_auth_obj, + model_max_budget_limiter=model_max_budget_limiter, + models=models, + request_data=request_data, + request=request, + llm_model_list=cast( # cast-ok: router model lists are deployment sequences + Sequence[object] | None, llm_model_list + ), + llm_router=llm_router, + project_object=typed_project_object, + team_object=typed_team_object, + end_user_object=end_user_object if isinstance(end_user_object, LiteLLM_EndUserTable) else None, + route=route, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) await _reserve_budget_after_common_checks( user_api_key_auth_obj=user_api_key_auth_obj, @@ -3184,7 +3798,10 @@ async def run_centralized_common_checks( request_data=request_data, route=route, llm_router=llm_router, - team_object=team_object, + team_object=cast( # cast-ok: preserve plain team-table objects through reservation + LiteLLM_TeamTableCachedObj | None, + team_object, + ), user_object=user_object, end_user_id=end_user_id, end_user_object=end_user_object, @@ -3855,13 +4472,13 @@ async def enforce_key_and_fallback_model_access( ) -_enforce_key_and_fallback_model_access: Final = enforce_key_and_fallback_model_access +_enforce_key_and_fallback_model_access: Final = enforce_key_and_fallback_model_access # pyright: ignore[reportUnknownVariableType] # existing tests use this alias async def _run_post_custom_auth_checks( valid_token: UserAPIKeyAuth, request: Request, - request_data: dict, + request_data: dict[str, object], route: str, parent_otel_span: Span | None, ) -> UserAPIKeyAuth: @@ -3956,7 +4573,9 @@ async def _run_post_custom_auth_checks( model_name=model_name, request_data=request_data, request=request, - llm_model_list=llm_model_list, + llm_model_list=cast( # cast-ok: router model lists are deployment sequences + Sequence[object] | None, llm_model_list + ), llm_router=llm_router, ) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 7f0e0b1fc87..ed9ac258e16 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -83,6 +83,7 @@ from litellm.litellm_core_utils.streaming_handler import ( from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_checks import ( can_key_call_resolved_model, + is_model_cost_zero, request_skips_budget_checks, tag_max_budget_check_for_tags, ) @@ -127,7 +128,7 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports ) from litellm.router import Router from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict -from litellm.router_utils.common_utils import resolve_model_group_alias +from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model from litellm.types.guardrails import GuardrailEventHooks from litellm.types.router import RouterRateLimitError from litellm.types.router_weights import validate_router_weights @@ -250,6 +251,7 @@ from litellm.proxy.litellm_pre_call_utils import ( add_litellm_data_to_request, refresh_proxy_server_request_body_snapshot, reject_url_valued_destination, + sanitize_for_log, ) from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval from litellm.types.utils import ( @@ -737,6 +739,7 @@ async def _resolve_per_request_model_group_alias( _REQUEST_MODEL: Final[TypeAdapter[str | list[str] | None]] = TypeAdapter(str | list[str] | None) +_ROUTER_SETTINGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) def _request_model(data: Mapping[str, object]) -> str | list[str] | None: @@ -1733,6 +1736,69 @@ def _timing_values( return getattr(logging_obj, "response_timing_metrics", None) or {} +async def _team_member_within_fallback_model_budget( + user_api_key_dict: UserAPIKeyAuth, + model: str, + llm_router: Router | None, + proxy_config: ProxyConfig | None, + proxy_logging_obj: ProxyLogging, +) -> bool: + member_caps: Final = user_api_key_dict.team_member_model_max_budget + user_id: Final = user_api_key_dict.user_id + team_id: Final = user_api_key_dict.team_id + if not isinstance(member_caps, Mapping) or not member_caps or user_id is None or team_id is None: + return True + from litellm.proxy.proxy_server import model_max_budget_limiter + + try: + from litellm.proxy.proxy_server import prisma_client + + router_settings: Final = ( + _ROUTER_SETTINGS_ADAPTER.validate_python( + await proxy_config.get_hierarchical_router_settings( # pyright: ignore[reportUnknownMemberType] # legacy bare-dict return + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + ) + ) + if proxy_config is not None and llm_router is not None + else None + ) + raw_aliases: Final[object] = cast( # cast-ok: auth aliases are dynamically typed at runtime + object, user_api_key_dict.aliases + ) + served_model: Final = resolve_served_model( + model, + litellm.model_alias_map, + raw_aliases, + router_settings.get("model_group_alias") if router_settings is not None else None, + ) + if is_model_cost_zero(model=served_model, llm_router=llm_router): + return True + budget_models: Final = ( + tuple(dict.fromkeys((model, served_model, llm_router.routable_model_group(served_model)))) + if llm_router is not None + else tuple(dict.fromkeys((model, served_model))) + ) + for budget_model in budget_models: + await model_max_budget_limiter.is_team_member_within_model_budget( + user_id=user_id, + team_id=team_id, + team_member_model_max_budget=member_caps, + model=budget_model, + ) + except litellm.BudgetExceededError: + return False + except Exception as e: # noqa: BLE001 # fail closed: a spend lookup failure must not bill the member + verbose_proxy_logger.warning( + "Skipping rate limit fallback to model=%s: member budget lookup failed: %s", + sanitize_for_log(model), + sanitize_for_log(e), + ) + return False + return True + + class ProxyBaseLLMRequestProcessing: def __init__(self, data: dict): self.data = data @@ -2371,6 +2437,14 @@ class ProxyBaseLLMRequestProcessing: for fallback_model in fallback_models: if fallback_model == original_model: continue + if not await _team_member_within_fallback_model_budget( + user_api_key_dict=user_api_key_dict, + model=fallback_model, + llm_router=llm_router, + proxy_config=proxy_config, + proxy_logging_obj=proxy_logging_obj, + ): + continue self.data = independent_snapshot(pristine) self.data["model"] = fallback_model try: diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index 8bca51784f2..eaa185917e5 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -4,7 +4,7 @@ import time from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Final +from typing import Final, cast # noqa: TID251 # untyped callback payload requires explicit narrowing from openai.types import Batch @@ -25,6 +25,7 @@ VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX: Final = "virtual_key_spend" END_USER_SPEND_CACHE_KEY_PREFIX: Final = "end_user_model_spend" USER_SPEND_CACHE_KEY_PREFIX: Final = "user_model_spend" TEAM_SPEND_CACHE_KEY_PREFIX: Final = "team_model_spend" +TEAM_MEMBER_SPEND_CACHE_KEY_PREFIX: Final = "team_member_model_spend" _SPEND_CACHE_KEY_PREFIXES: Final = MappingProxyType( { @@ -32,6 +33,7 @@ _SPEND_CACHE_KEY_PREFIXES: Final = MappingProxyType( Litellm_EntityType.USER: USER_SPEND_CACHE_KEY_PREFIX, Litellm_EntityType.END_USER: END_USER_SPEND_CACHE_KEY_PREFIX, Litellm_EntityType.TEAM: TEAM_SPEND_CACHE_KEY_PREFIX, + Litellm_EntityType.TEAM_MEMBER: TEAM_MEMBER_SPEND_CACHE_KEY_PREFIX, } ) @@ -45,6 +47,7 @@ _BUDGET_START_TIME_KEY_PREFIXES: Final = MappingProxyType( Litellm_EntityType.USER: "user_model_budget_start_time", Litellm_EntityType.END_USER: "end_user_budget_start_time", Litellm_EntityType.TEAM: "team_model_budget_start_time", + Litellm_EntityType.TEAM_MEMBER: "team_member_model_budget_start_time", } ) @@ -63,6 +66,10 @@ class ResolvedModelBudget: budget_config: BudgetConfig +def team_member_budget_entity_id(user_id: str, team_id: str) -> str: + return f"{len(user_id)}:{user_id}:{team_id}" + + def model_budget_spend_cache_key( entity_type: Litellm_EntityType, entity_id: str | None, @@ -290,6 +297,18 @@ def _resolve_entity_model_budgets( ) +def _metadata_value(metadata: object, key: str) -> object | None: + metadata_mapping: Final[Mapping[str, object]] = cast( # cast-ok: callback metadata has string keys + Mapping[str, object], metadata + ) + return metadata_mapping.get(key) + + +def _metadata_string_value(metadata: object, key: str) -> str | None: + value: Final = _metadata_value(metadata, key) + return value if isinstance(value, str) else None + + class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): """ Handles budgets for model + virtual key @@ -410,6 +429,24 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): exceeded_message=f"LiteLLM Team: {team_id}, exceeded budget for model={model}", ) + @with_service_target("model_budgets") + async def is_team_member_within_model_budget( + self, + user_id: str, + team_id: str, + team_member_model_max_budget: Mapping[str, object], + model: str, + ) -> bool: + return await self._is_entity_within_model_budget( + entity_type=Litellm_EntityType.TEAM_MEMBER, + entity_id=team_member_budget_entity_id(user_id=user_id, team_id=team_id), + model_max_budget=team_member_model_max_budget, + model=model, + exceeded_message=( + f"LiteLLM Team Member: user={user_id}, team={team_id}, exceeded budget for model={model}" + ), + ) + async def _is_entity_within_model_budget( self, entity_type: Litellm_EntityType, @@ -523,6 +560,19 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): response_cost: Final[float] = standard_logging_payload.get("response_cost", 0) key_model_max_budget: Final = _metadata.get("user_api_key_model_max_budget") + team_member_user_id: Final = _metadata_string_value( + payload_metadata, # pyright: ignore[reportUnknownArgumentType] # callback metadata is partially untyped + "user_api_key_user_id", + ) + team_member_team_id: Final = _metadata_string_value( + payload_metadata, # pyright: ignore[reportUnknownArgumentType] # callback metadata is partially untyped + "user_api_key_team_id", + ) + team_member_entity_id: Final = ( + team_member_budget_entity_id(user_id=team_member_user_id, team_id=team_member_team_id) + if team_member_user_id is not None and team_member_team_id is not None + else None + ) entity_budgets: Final = ( ( Litellm_EntityType.KEY, @@ -553,6 +603,14 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): standard_logging_payload.get("end_user") or payload_metadata.get("user_api_key_end_user_id"), _metadata.get("user_api_key_end_user_model_max_budget"), ), + ( + Litellm_EntityType.TEAM_MEMBER, + team_member_entity_id, + _metadata_value( + _metadata, # pyright: ignore[reportUnknownArgumentType] # callback metadata is partially untyped + "user_api_key_team_member_model_max_budget", + ), + ), ) resolved_budgets: Final = _resolve_entity_model_budgets(model=model, entity_budgets=entity_budgets) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 913310952cd..0d693b85ce8 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2508,6 +2508,9 @@ async def add_litellm_data_to_request( data[_metadata_variable_name]["user_api_key_user_max_budget"] = user_api_key_dict.user_max_budget user_model_budget: Final = user_api_key_dict.user_model_max_budget data[_metadata_variable_name]["user_api_key_user_model_max_budget"] = user_model_budget # rebind-ok: out-param + data[_metadata_variable_name][ # rebind-ok: adds member budget for spend tracking + "user_api_key_team_member_model_max_budget" + ] = user_api_key_dict.team_member_model_max_budget data[_metadata_variable_name].update(carried_budget_metadata(user_api_key_dict)) data[_metadata_variable_name]["user_api_key_metadata"] = strip_callback_config(user_api_key_dict.metadata) diff --git a/litellm/proxy/management_endpoints/budget_management_endpoints.py b/litellm/proxy/management_endpoints/budget_management_endpoints.py index d001dc666af..b3f372327dc 100644 --- a/litellm/proxy/management_endpoints/budget_management_endpoints.py +++ b/litellm/proxy/management_endpoints/budget_management_endpoints.py @@ -19,6 +19,8 @@ from typing import Final from fastapi import APIRouter, Depends, HTTPException +from litellm._internal_context import service_target +from litellm._logging import verbose_proxy_logger from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time @@ -141,7 +143,7 @@ async def update_budget( - model_max_budget: Optional[dict] - Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d", "tpm_limit": 100000, "rpm_limit": 100000}} - budget_reset_at: Optional[datetime] - Update the Datetime when the budget was last reset. """ - from litellm.proxy.proxy_server import litellm_proxy_admin_name, prisma_client + from litellm.proxy.proxy_server import litellm_proxy_admin_name, prisma_client, user_api_key_cache if prisma_client is None: raise HTTPException( @@ -199,6 +201,11 @@ async def update_budget( where={"budget_id": budget_obj.budget_id}, data=budget_obj_jsonified, ) + with service_target("team_member_budgets"): + try: + await user_api_key_cache.async_delete_cache(key=f"team_member_default_budget:{budget_obj.budget_id}") + except Exception: # noqa: BLE001 # Cache failure must not fail a committed budget update. + verbose_proxy_logger.warning("Failed to evict the team member default budget cache", exc_info=True) return response diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index 6a70a0137a4..adf89c29279 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -1,3 +1,4 @@ +import json import math from collections.abc import Mapping from types import MappingProxyType @@ -580,21 +581,31 @@ MEMBER_BUDGET_PATCH_FIELDS: Final = MappingProxyType( "allowed_models": "allowed_models", "temp_budget_increase": "temp_budget_increase", "temp_budget_expiry": "temp_budget_expiry", + "model_max_budget": "model_max_budget", } ) -def _prisma_value(value: object) -> object: +_JSON_BUDGET_COLUMNS: Final = frozenset({"model_max_budget"}) + + +def _prisma_value(column: str, value: object) -> object: + if column in _JSON_BUDGET_COLUMNS: + return json.dumps(value if value is not None else {}) return list(value) if isinstance(value, tuple) else value +def _prisma_payload(data: Mapping[str, object]) -> dict[str, object]: + return {column: _prisma_value(column, value) for column, value in data.items()} + + def member_budget_patch(source: BaseModel) -> Mapping[str, object]: """Map the per-member limit fields a request actually set to their budget-table columns (merge-patch: a sent value updates, an explicit null clears, an absent field is left untouched).""" provided: Final = source.model_dump(exclude_unset=True) return { - column: _prisma_value(provided[request_field]) + column: provided[request_field] for request_field, column in MEMBER_BUDGET_PATCH_FIELDS.items() if request_field in provided } @@ -603,7 +614,7 @@ def member_budget_patch(source: BaseModel) -> Mapping[str, object]: def _is_set_budget_value(value: object) -> bool: if value is None: return False - if isinstance(value, list) and len(value) == 0: + if isinstance(value, (list, dict)) and len(value) == 0: return False return True @@ -645,10 +656,26 @@ async def upsert_budget_and_membership( A patch that only touches the temporary budget pair never copies permanent limits into a new row, so the member keeps inheriting the live team default. """ - if not budget_patch: + ignore_default_model_budget_clear: Final = ( + existing_budget_id is not None + and existing_budget_id == team_default_budget_id + and "model_max_budget" in budget_patch + and ( + budget_patch["model_max_budget"] is None + or (isinstance(budget_patch["model_max_budget"], Mapping) and not budget_patch["model_max_budget"]) + ) + ) + effective_patch: Final = MappingProxyType( + { + field: value + for field, value in budget_patch.items() + if not (ignore_default_model_budget_clear and field == "model_max_budget") + } + ) + if not effective_patch: return - write_data: Final = dict(budget_patch) + write_data: Final = dict(effective_patch) if "budget_duration" in write_data: duration: Final = write_data["budget_duration"] write_data["budget_reset_at"] = ( @@ -675,7 +702,7 @@ async def upsert_budget_and_membership( return await tx.litellm_budgettable.update( where={"budget_id": existing_budget_id}, - data={"updated_by": user_api_key_dict.user_id or "", **write_data}, + data={"updated_by": user_api_key_dict.user_id or "", **_prisma_payload(write_data)}, ) return @@ -689,14 +716,24 @@ async def upsert_budget_and_membership( create_data: Final[dict[str, object]] = { # mutable-ok: Prisma create payloads are dict-shaped "created_by": user_api_key_dict.user_id or "", "updated_by": user_api_key_dict.user_id or "", + "model_max_budget": {}, **MappingProxyType( - {f: source[f] for f in _TEAM_MEMBER_BUDGET_LIMIT_FIELDS if _is_set_budget_value(source.get(f))} + { + f: source[f] + for f in _TEAM_MEMBER_BUDGET_LIMIT_FIELDS + if ( + f != "model_max_budget" + or team_default_budget_id is None + or source.get("budget_id") != team_default_budget_id + ) + and _is_set_budget_value(source.get(f)) + } ), **write_data, } # Restarting an inherited window on an unrelated edit hands the member a free period. - carried: Final = source.get("budget_reset_at") if "budget_duration" not in budget_patch else None + carried: Final = source.get("budget_reset_at") if "budget_duration" not in effective_patch else None if carried is not None: create_data["budget_reset_at"] = carried if create_data.get("budget_reset_at") is None: @@ -708,7 +745,7 @@ async def upsert_budget_and_membership( return new_budget: Final = await tx.litellm_budgettable.create( - data=create_data, + data=_prisma_payload(create_data), include={"team_membership": True}, ) await tx.litellm_teammembership.upsert( diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index c78434eda83..544bc471ce5 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -37,6 +37,7 @@ from pydantic import BaseModel, Field, JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict, assert_never import litellm +from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.integrations.prometheus import PrometheusLogger @@ -228,7 +229,7 @@ from litellm.types.proxy.management_endpoints.team_endpoints import ( TeamUserSpendRow, UpdateTeamMemberPermissionsRequest, ) -from litellm.types.utils import BudgetConfig +from litellm.types.utils import BudgetConfig, GenericBudgetConfigType if TYPE_CHECKING: from prisma import Prisma @@ -547,6 +548,7 @@ class TeamMemberBudgetHandler: team_member_rpm_limit: int | None = None, team_member_tpm_limit: int | None = None, team_member_budget_duration: str | None = None, + team_member_model_max_budget: GenericBudgetConfigType | None = None, ) -> bool: """Check if any team member limits are provided""" return any( @@ -555,9 +557,22 @@ class TeamMemberBudgetHandler: team_member_rpm_limit is not None, team_member_tpm_limit is not None, team_member_budget_duration is not None, + team_member_model_max_budget is not None, ] ) + @staticmethod + @with_service_target("team_member_budgets") + async def _evict_default_budget_cache( + budget_id: str, + user_api_key_cache: UserApiKeyCache | None, + ) -> None: + if user_api_key_cache is not None: + try: + await user_api_key_cache.async_delete_cache(key=f"team_member_default_budget:{budget_id}") + except Exception: # noqa: BLE001 # Cache failure must not fail a committed budget update. + verbose_proxy_logger.warning("Failed to evict the team member default budget cache", exc_info=True) + @staticmethod async def create_team_member_budget_table( data: NewTeamRequest | _TeamAliasBudgetRow, @@ -567,6 +582,8 @@ class TeamMemberBudgetHandler: team_member_rpm_limit: int | None = None, team_member_tpm_limit: int | None = None, team_member_budget_duration: str | None = None, + team_member_model_max_budget: GenericBudgetConfigType | None = None, + user_api_key_cache: UserApiKeyCache | None = None, explicitly_set_fields: AbstractSet[str] = frozenset(), ) -> dict: """Create team member budget table with provided limits. @@ -602,11 +619,17 @@ class TeamMemberBudgetHandler: budget_request.tpm_limit = team_member_tpm_limit if team_member_budget_duration is not None: budget_request.budget_duration = team_member_budget_duration + if team_member_model_max_budget is not None: + budget_request.model_max_budget = team_member_model_max_budget team_member_budget_table: Final = await _as_budget_write(new_budget)( budget_obj=budget_request, user_api_key_dict=user_api_key_dict, ) + await TeamMemberBudgetHandler._evict_default_budget_cache( + budget_id=team_member_budget_table.budget_id, + user_api_key_cache=user_api_key_cache, + ) # Add team_member_budget_id as metadata field to team table if new_team_data_json.get("metadata") is None: @@ -627,6 +650,8 @@ class TeamMemberBudgetHandler: team_member_rpm_limit: int | None = None, team_member_tpm_limit: int | None = None, team_member_budget_duration: str | None = None, + team_member_model_max_budget: GenericBudgetConfigType | None = None, + user_api_key_cache: UserApiKeyCache | None = None, explicitly_set_fields: AbstractSet[str] = frozenset(), ) -> dict: """Upsert team member budget table with provided limits. @@ -657,11 +682,19 @@ class TeamMemberBudgetHandler: budget_request.budget_duration = team_member_budget_duration if team_member_budget_duration is None: budget_request.budget_reset_at = None + if team_member_model_max_budget is not None or "team_member_model_max_budget" in explicitly_set_fields: + budget_request.model_max_budget = ( + {} if team_member_model_max_budget is None else team_member_model_max_budget + ) budget_row: Final = await _as_budget_write(update_budget)( budget_obj=budget_request, user_api_key_dict=user_api_key_dict, ) + await TeamMemberBudgetHandler._evict_default_budget_cache( + budget_id=team_member_budget_id, + user_api_key_cache=user_api_key_cache, + ) verbose_proxy_logger.info( "Updated team member budget table: %s, with team_member_budget=%s, team_member_rpm_limit=%s, team_member_tpm_limit=%s", budget_row.budget_id, @@ -682,6 +715,8 @@ class TeamMemberBudgetHandler: team_member_rpm_limit=team_member_rpm_limit, team_member_tpm_limit=team_member_tpm_limit, team_member_budget_duration=team_member_budget_duration, + team_member_model_max_budget=team_member_model_max_budget, + user_api_key_cache=user_api_key_cache, explicitly_set_fields=explicitly_set_fields, ) @@ -696,6 +731,9 @@ class TeamMemberBudgetHandler: data_dict.pop("team_member_budget_duration", None) data_dict.pop("team_member_rpm_limit", None) data_dict.pop("team_member_tpm_limit", None) + data_dict.pop( # pyright: ignore[reportUnknownMemberType] # legacy raw dict lacks key typing + "team_member_model_max_budget", None + ) _clean_team_member_fields = clean_team_member_fields @@ -705,6 +743,7 @@ class TeamMemberBudgetHandler: user_api_key_dict: "UserAPIKeyAuth", updated_kv: dict, explicitly_set_fields: set, + user_api_key_cache: UserApiKeyCache | None = None, ) -> dict: """Clear explicitly-nulled fields on the team member budget row.""" from litellm.proxy._types import BudgetNewRequest @@ -727,10 +766,16 @@ class TeamMemberBudgetHandler: budget_request.rpm_limit = None if "team_member_tpm_limit" in explicitly_set_fields: budget_request.tpm_limit = None + if "team_member_model_max_budget" in explicitly_set_fields: + budget_request.model_max_budget = {} await update_budget( budget_obj=budget_request, user_api_key_dict=user_api_key_dict, ) + await TeamMemberBudgetHandler._evict_default_budget_cache( + budget_id=team_member_budget_id, + user_api_key_cache=user_api_key_cache, + ) TeamMemberBudgetHandler.clean_team_member_fields(updated_kv) return updated_kv @@ -1420,6 +1465,7 @@ async def new_team( - object_permission: Optional[LiteLLM_ObjectPermissionBase] - team-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"], "agents": ["agent_1", "agent_2"], "agent_access_groups": ["dev_group"]}. IF null or {} then no object permission. - team_member_budget: Optional[float] - The maximum budget allocated to an individual team member. - team_member_budget_duration: Optional[str] - The duration of the budget for the team member. Doc [here](https://docs.litellm.ai/docs/proxy/team_budgets) + - team_member_model_max_budget: Optional[dict] - Per-model budgets shared by team members without an override. - team_member_rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for individual team members. - team_member_tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for individual team members. - team_member_key_duration: Optional[str] - The duration for a team member's key. e.g. "1d", "1w", "1mo" @@ -1508,6 +1554,10 @@ async def new_team( validate_budget_duration(data.budget_duration) validate_budget_duration(data.team_member_budget_duration) validate_team_model_max_budget(model_max_budget=data.model_max_budget, premium_user=premium_user) + validate_team_model_max_budget( + model_max_budget=data.team_member_model_max_budget, + premium_user=premium_user, + ) if data.soft_budget is not None: if data.max_budget is not None: @@ -1702,6 +1752,7 @@ async def new_team( team_member_rpm_limit=data.team_member_rpm_limit, team_member_tpm_limit=data.team_member_tpm_limit, team_member_budget_duration=data.team_member_budget_duration, + team_member_model_max_budget=data.team_member_model_max_budget, ): data_json = await TeamMemberBudgetHandler.create_team_member_budget_table( data=data, @@ -1711,6 +1762,8 @@ async def new_team( team_member_rpm_limit=data.team_member_rpm_limit, team_member_tpm_limit=data.team_member_tpm_limit, team_member_budget_duration=data.team_member_budget_duration, + team_member_model_max_budget=data.team_member_model_max_budget, + user_api_key_cache=user_api_key_cache, explicitly_set_fields=data.model_fields_set, ) @@ -2190,6 +2243,7 @@ async def update_team( - object_permission: Optional[LiteLLM_ObjectPermissionBase] - team-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"], "agents": ["agent_1", "agent_2"], "agent_access_groups": ["dev_group"]}. IF null or {} then no object permission. - team_member_budget: Optional[float] - The maximum budget allocated to an individual team member. - team_member_budget_duration: Optional[str] - The duration of the budget for the team member. Doc [here](https://docs.litellm.ai/docs/proxy/team_budgets) + - team_member_model_max_budget: Optional[dict] - Per-model budgets shared by team members without an override. - team_member_rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for individual team members. - team_member_tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for individual team members. - team_member_key_duration: Optional[str] - The duration for a team member's key. e.g. "1d", "1w", "1mo" @@ -2276,6 +2330,10 @@ async def update_team( validate_budget_duration(data.budget_duration) validate_budget_duration(data.team_member_budget_duration) validate_team_model_max_budget(model_max_budget=data.model_max_budget, premium_user=premium_user) + validate_team_model_max_budget( + model_max_budget=data.team_member_model_max_budget, + premium_user=premium_user, + ) existing_team_row = await _raw_team_db(TeamRepository(prisma_client)).find_unique( where={"team_id": data.team_id} @@ -2495,6 +2553,7 @@ async def update_team( "team_member_rpm_limit", "team_member_tpm_limit", "team_member_budget_duration", + "team_member_model_max_budget", ] if field in updated_kv } @@ -2524,6 +2583,7 @@ async def update_team( team_member_rpm_limit=data.team_member_rpm_limit, team_member_tpm_limit=data.team_member_tpm_limit, team_member_budget_duration=data.team_member_budget_duration, + team_member_model_max_budget=data.team_member_model_max_budget, ): updated_kv = await TeamMemberBudgetHandler.upsert_team_member_budget_table( team_table=existing_team_row, @@ -2533,6 +2593,8 @@ async def update_team( team_member_rpm_limit=data.team_member_rpm_limit, team_member_tpm_limit=data.team_member_tpm_limit, team_member_budget_duration=data.team_member_budget_duration, + team_member_model_max_budget=data.team_member_model_max_budget, + user_api_key_cache=user_api_key_cache, explicitly_set_fields=_team_member_fields_in_request, ) # Backfill team_memberships for members who joined before the @@ -2556,6 +2618,7 @@ async def update_team( user_api_key_dict=user_api_key_dict, updated_kv=updated_kv, explicitly_set_fields=_team_member_fields_in_request, + user_api_key_cache=user_api_key_cache, ) else: TeamMemberBudgetHandler.clean_team_member_fields(updated_kv) @@ -3866,6 +3929,7 @@ async def team_member_update( ) validate_budget_duration(data.budget_duration) + validate_team_model_max_budget(data.model_max_budget, premium_user=premium_user) _existing_team_row: Final = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id}) @@ -3982,6 +4046,7 @@ async def team_member_update( allowed_models=data.allowed_models, temp_budget_increase=data.temp_budget_increase, temp_budget_expiry=data.temp_budget_expiry, + model_max_budget=data.model_max_budget, ) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 5d73a04f9e0..d6358c638e7 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -677,6 +677,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): _metadata["user_api_key_model_max_budget"] = user_api_key_dict.model_max_budget _metadata["user_api_key_team_model_max_budget"] = user_api_key_dict.team_model_max_budget _metadata["user_api_key_user_model_max_budget"] = user_api_key_dict.user_model_max_budget + _metadata["user_api_key_team_member_model_max_budget"] = user_api_key_dict.team_member_model_max_budget _metadata["user_api_key_end_user_model_max_budget"] = user_api_key_dict.end_user_model_max_budget else: for field in ( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index dac46855fd6..089647c6297 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -359,7 +359,7 @@ from litellm.proxy.auth.auth_utils import ( log_once_if_budget_reservation_disabled, warn_once_if_custom_auth_skips_common_checks, ) -from litellm.proxy.auth.fallback_budget import router_fallback_budget_check +from litellm.proxy.auth.fallback_budget import RouterFallbackBudgetCheck, router_fallback_budget_check from litellm.proxy.auth.fallback_model_access import router_fallback_access_check from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY, LicenseCheck @@ -7196,7 +7196,10 @@ class ProxyConfig: ), ignore_invalid_deployments=True, # don't raise an error if a deployment is invalid fallback_access_check=router_fallback_access_check, - fallback_budget_check=router_fallback_budget_check, + fallback_budget_check=RouterFallbackBudgetCheck( + is_enforced=router_fallback_budget_check.is_enforced, + team_member_model_budget_limiter=model_max_budget_limiter, + ), auto_router_capability_limit=_license_check.auto_router_capability_limit, ) @@ -7691,7 +7694,10 @@ class ProxyConfig: search_tools=search_tools, ignore_invalid_deployments=True, fallback_access_check=router_fallback_access_check, - fallback_budget_check=router_fallback_budget_check, + fallback_budget_check=RouterFallbackBudgetCheck( + is_enforced=router_fallback_budget_check.is_enforced, + team_member_model_budget_limiter=model_max_budget_limiter, + ), auto_router_capability_limit=_license_check.auto_router_capability_limit, ) verbose_proxy_logger.debug("updated llm_router: %s", llm_router) diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 4d6af710bb9..6976a64ef39 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -4,6 +4,8 @@ from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, TypeVar +from pydantic import TypeAdapter + if TYPE_CHECKING: from litellm.types.llms.openai import OpenAIFileObject @@ -19,6 +21,7 @@ from litellm.types.utils import LlmProviders _V = TypeVar("_V") ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors"}) +_OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) def without_router_only_kwargs(kwargs: Mapping[str, _V]) -> dict[str, _V]: @@ -72,6 +75,30 @@ def resolve_model_group_alias(model_group_alias: object, model: str) -> str | No return target +def resolve_served_model( + model: str, + model_alias_map: object, + key_aliases: object, + model_group_alias: object, +) -> str: + """Model the request preparation serves for `model`. + + Apply the global alias map, then key aliases, then `router_settings.model_group_alias`. + """ + global_aliases: Final = ( + _OBJECT_MAPPING_ADAPTER.validate_python(model_alias_map) if isinstance(model_alias_map, Mapping) else None + ) + global_target: Final = global_aliases.get(model) if global_aliases is not None else None + after_global: Final = global_target if isinstance(global_target, str) else model + key_alias_mapping: Final = ( + _OBJECT_MAPPING_ADAPTER.validate_python(key_aliases) if isinstance(key_aliases, Mapping) else None + ) + key_target: Final = key_alias_mapping.get(after_global) if key_alias_mapping is not None else None + after_key: Final = key_target if isinstance(key_target, str) else after_global + router_target: Final = resolve_model_group_alias(model_group_alias, after_key) + return router_target if router_target is not None else after_key + + def truncate_fallback_error_detail(detail: str) -> str: """ Bound a fallback failure detail before it is logged or appended to an exception message. diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py b/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py index ced53794cc2..494e742e426 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py +++ b/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py @@ -13,7 +13,7 @@ Coverage: """ import json -from typing import Any, Dict, List +from typing import Any, Dict, Final, List from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -1198,6 +1198,7 @@ def _fake_user_api_key_auth( end_user_model_max_budget=None, end_user_id=None, user_model_max_budget=None, + team_member_model_max_budget=None, user_id=None, token=None, ): @@ -1219,6 +1220,7 @@ def _fake_user_api_key_auth( auth.end_user_model_max_budget = end_user_model_max_budget auth.end_user_id = end_user_id auth.user_model_max_budget = user_model_max_budget + auth.team_member_model_max_budget = team_member_model_max_budget auth.user_id = user_id auth.token = token return auth @@ -1659,6 +1661,120 @@ async def test_summary_model_denied_when_user_over_model_budget(): assert kwarg in real_params, f"compact.py passes {kwarg}=, which the limiter no longer accepts" +async def test_summary_model_denied_when_team_member_over_model_budget(monkeypatch: pytest.MonkeyPatch): + from litellm.caching.dual_cache import DualCache + from litellm.llms.anthropic.pass_through.context_management.constants import COMPACT_SUMMARY_MODEL_SETTING_KEY + from litellm.models.team_membership import LiteLLM_TeamMembership + from litellm.proxy._types import ( + LiteLLM_BudgetTable, + LiteLLM_TeamTableCachedObj, + LitellmUserRoles, + UserAPIKeyAuth, + ) + from litellm.proxy.auth.auth_checks import get_team_member_default_budget + from litellm.proxy.auth.user_api_key_auth import _load_team_member_default_model_budget + from litellm.proxy.common_utils.user_api_key_cache import ( + UserApiKeyCache, + team_membership_reservation_cache_key, + ) + from litellm.proxy.hooks.model_max_budget_limiter import PROXY_VirtualKeyModelMaxBudgetLimiter + from litellm.proxy.proxy_server import general_settings + + import litellm.proxy.proxy_server as proxy_server + + budget_id: Final = "summary-budget" + member_budget: Final = {"claude-haiku-4-5": {"max_budget": 0.0, "budget_duration": "1d"}} + user_id: Final = "member-over-budget" + team_id: Final = "team-1" + user_api_key_cache: Final = UserApiKeyCache() + prisma_client: Final = MagicMock() + team_object: Final = LiteLLM_TeamTableCachedObj( + team_id=team_id, + models=["all-proxy-models"], + metadata={"team_member_budget_id": budget_id}, + ) + membership: Final = LiteLLM_TeamMembership( + user_id=user_id, + team_id=team_id, + budget_id=budget_id, + litellm_budget_table=LiteLLM_BudgetTable(budget_id=budget_id, model_max_budget={}), + ) + await user_api_key_cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id=user_id, team_id=team_id), + value=membership, + model_type=LiteLLM_TeamMembership, + ) + await user_api_key_cache.async_set_cache( + key=f"team_member_default_budget:{budget_id}", + value=LiteLLM_BudgetTable(budget_id=budget_id, model_max_budget=member_budget), + model_type=LiteLLM_BudgetTable, + ) + await user_api_key_cache.async_set_cache( + key=f"team_id:{team_id}", + value=team_object, + model_type=LiteLLM_TeamTableCachedObj, + ) + + auth: Final = UserAPIKeyAuth( + token="hashed-token", + user_id=user_id, + team_id=team_id, + user_role=LitellmUserRoles.INTERNAL_USER, + models=["all-proxy-models"], + ) + await _load_team_member_default_model_budget( + user_api_key_auth_obj=auth, + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + skip_budget_checks=False, + proxy_logging_obj=None, + ) + assert auth.team_member_model_max_budget == member_budget + assert ( + await get_team_member_default_budget( + budget_id=budget_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + raise_on_lookup_error=True, + ) + ).model_max_budget == member_budget + + router: Final = litellm.Router( + model_list=[ + {"model_name": MODEL, "litellm_params": {"model": MODEL}}, + { + "model_name": "claude-haiku-4-5", + "litellm_params": { + "model": "claude-haiku-4-5", + "mock_response": "x", + }, + }, + ] + ) + limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache()) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "user_api_key_cache", user_api_key_cache) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "model_max_budget_limiter", limiter) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock()) + monkeypatch.setitem(general_settings, COMPACT_SUMMARY_MODEL_SETTING_KEY, "claude-haiku-4-5") + messages: Final = [{"role": "user", "content": "hello " * 60_000}] + + result: Final = await apply_compact_20260112( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 50_000}}, + user_api_key_auth=auth, + llm_router=router, + ) + + assert result.applied_edits[0].get("error") == "summary_model_budget_exceeded" + assert result.compaction_block is None + + async def test_summary_model_denied_when_end_user_over_model_budget(): """End-user per-model budget is enforced for the summary subrequest too.""" import litellm @@ -2022,9 +2138,7 @@ async def test_summary_model_denied_when_team_over_model_budget(): limiter = MagicMock() limiter.is_key_within_model_budget = AsyncMock(return_value=True) limiter.is_team_within_model_budget = AsyncMock( - side_effect=litellm.BudgetExceededError( - message="over budget", current_cost=10, max_budget=5 - ) + side_effect=litellm.BudgetExceededError(message="over budget", current_cost=10, max_budget=5) ) with ( @@ -2064,9 +2178,7 @@ async def test_summary_model_denied_when_team_over_model_budget(): PROXY_VirtualKeyModelMaxBudgetLimiter, ) - real_params = inspect.signature( - PROXY_VirtualKeyModelMaxBudgetLimiter.is_team_within_model_budget - ).parameters + real_params = inspect.signature(PROXY_VirtualKeyModelMaxBudgetLimiter.is_team_within_model_budget).parameters for kwarg in ("team_id", "team_model_max_budget", "key_model_max_budget", "model"): assert kwarg in real_params, f"compact.py passes {kwarg}=, which the limiter does not accept" @@ -2585,6 +2697,9 @@ async def test_prepare_context_managed_request_forwards_proxy_litellm_metadata() "user_api_key": "sk-parent", "user_api_key_team_id": "team-abc", "user_api_key_user_id": "user-xyz", + "user_api_key_team_member_model_max_budget": { + "claude-haiku-4-5": {"budget_limit": 4.0, "time_period": "1d"} + }, "litellm_call_id": "call-1", }, additional_drop_params=None, @@ -2595,6 +2710,9 @@ async def test_prepare_context_managed_request_forwards_proxy_litellm_metadata() assert captured_summary_metadata.get("user_api_key") == "sk-parent" assert captured_summary_metadata.get("user_api_key_team_id") == "team-abc" assert captured_summary_metadata.get("user_api_key_user_id") == "user-xyz" + assert captured_summary_metadata.get("user_api_key_team_member_model_max_budget") == { + "claude-haiku-4-5": {"budget_limit": 4.0, "time_period": "1d"} + } assert captured_summary_metadata.get("litellm_call_id") == "call-1" # Anthropic-shape ``metadata.user_id`` must not leak in as a propagated field. assert "user_id" not in captured_summary_metadata diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index e26845e5b2a..8b712772230 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -7703,6 +7703,47 @@ async def test_get_team_member_default_budget_caches_json_safe_payload(): mock_prisma_client.db.litellm_budgettable.find_unique.assert_awaited_once() +@pytest.mark.asyncio +async def test_get_team_member_default_budget_reraises_lookup_error_when_requested() -> None: + from litellm.proxy.auth.auth_checks import get_team_member_default_budget + + lookup_error: Final = RuntimeError("budget database unavailable") + cache: Final = MagicMock() + cache.async_get_cache = AsyncMock(return_value=None) + prisma_client: Final = MagicMock() + prisma_client.db.litellm_budgettable.find_unique = AsyncMock(side_effect=lookup_error) + + with pytest.raises(RuntimeError) as exc_info: + await get_team_member_default_budget( + budget_id="default-budget", + prisma_client=prisma_client, + user_api_key_cache=cache, + raise_on_lookup_error=True, + ) + + assert exc_info.value is lookup_error + + +@pytest.mark.asyncio +async def test_get_team_member_default_budget_returns_none_on_lookup_error_by_default() -> None: + from litellm.proxy.auth.auth_checks import get_team_member_default_budget + + prisma_client: Final = MagicMock() + prisma_client.db.litellm_budgettable.find_unique = AsyncMock( + side_effect=RuntimeError("budget database unavailable") + ) + cache: Final = MagicMock() + cache.async_get_cache = AsyncMock(return_value=None) + + budget: Final = await get_team_member_default_budget( + budget_id="default-budget", + prisma_client=prisma_client, + user_api_key_cache=cache, + ) + + assert budget is None + + @pytest.mark.asyncio async def test_get_end_user_object_db_fetch_returns_validated_end_user(): from litellm.proxy.auth.auth_checks import get_end_user_object diff --git a/tests/unit/proxy/auth/test_fallback_budget.py b/tests/unit/proxy/auth/test_fallback_budget.py index 00c1a7cdefc..3d31f1e0c78 100644 --- a/tests/unit/proxy/auth/test_fallback_budget.py +++ b/tests/unit/proxy/auth/test_fallback_budget.py @@ -1,12 +1,23 @@ +from collections.abc import Mapping, Sequence +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +import litellm import pytest from litellm import Router -from litellm.proxy._types import UserAPIKeyAuth +from litellm.caching.dual_cache import DualCache +from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth from litellm.proxy.auth.fallback_budget import ( RouterFallbackBudgetCheck, is_token_within_budget_for_model, router_fallback_budget_check, ) +from litellm.proxy.hooks.model_max_budget_limiter import ( + PROXY_VirtualKeyModelMaxBudgetLimiter, + model_budget_spend_cache_key, + team_member_budget_entity_id, +) FREE_MODEL = { "model_name": "free-model", @@ -48,6 +59,41 @@ def _token(**overrides) -> UserAPIKeyAuth: return UserAPIKeyAuth(**fields) +async def _team_member_limiter( + spend_by_model: Mapping[str, float] | None = None, +) -> tuple[PROXY_VirtualKeyModelMaxBudgetLimiter, MagicMock]: + entity_id: Final = team_member_budget_entity_id(user_id="u1", team_id="t1") + spend_by_key: Final = { + model_budget_spend_cache_key( + entity_type=Litellm_EntityType.TEAM_MEMBER, + entity_id=entity_id, + budget_model=model, + budget_duration="1d", + ): spend + for model, spend in (spend_by_model or {}).items() + } + redis_cache: Final = MagicMock() + + async def get_spend(*, key: str) -> object | None: + return spend_by_key.get(key) + + redis_cache.async_get_cache = AsyncMock(side_effect=get_spend) + return PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache(redis_cache=redis_cache)), redis_cache + + +def _member_budget_spend_keys(models: Sequence[str]) -> tuple[str, ...]: + entity_id: Final = team_member_budget_entity_id(user_id="u1", team_id="t1") + return tuple( + model_budget_spend_cache_key( + entity_type=Litellm_EntityType.TEAM_MEMBER, + entity_id=entity_id, + budget_model=model, + budget_duration="1d", + ) + for model in models + ) + + ENFORCED = RouterFallbackBudgetCheck(is_enforced=lambda: True) NOT_ENFORCED = RouterFallbackBudgetCheck(is_enforced=lambda: False) @@ -70,6 +116,201 @@ async def test_paid_target_refused_when_over_user_budget(): assert await is_token_within_budget_for_model(model="paid-model", valid_token=token, llm_router=_router()) is False +@pytest.mark.asyncio +async def test_team_member_budget_refuses_an_exhausted_paid_target() -> None: + member_caps: Final = {"paid-model": {"max_budget": 5.0, "budget_duration": "1d"}} + token: Final = _token( + user_id="u1", + team_id="t1", + team_member_model_max_budget=member_caps, + ) + limiter, redis_cache = await _team_member_limiter(spend_by_model={"paid-model": 10.0}) + + within_budget: Final = await is_token_within_budget_for_model( + model="paid-model", + valid_token=token, + llm_router=_router(), + team_member_model_budget_limiter=limiter, + ) + + assert within_budget is False + assert tuple(call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list) == ( + _member_budget_spend_keys(("paid-model",)) + ) + + +@pytest.mark.asyncio +async def test_router_fallback_budget_check_resolves_model_group_alias_before_member_cap_check() -> None: + expensive_model: Final = { + **PAID_MODEL, + "model_name": "expensive", + "model_info": {"id": "expensive-model-id"}, + } + router: Final = Router( + model_list=[FREE_MODEL, expensive_model], + model_group_alias={"backup": "expensive"}, + ) + member_caps: Final = { + "backup": {"max_budget": 100.0, "budget_duration": "1d"}, + "expensive": {"max_budget": 5.0, "budget_duration": "1d"}, + } + token: Final = _token(user_id="u1", team_id="t1", team_member_model_max_budget=member_caps) + limiter, redis_cache = await _team_member_limiter(spend_by_model={"backup": 1.0, "expensive": 10.0}) + budget_check: Final = RouterFallbackBudgetCheck( + is_enforced=lambda: True, + team_member_model_budget_limiter=limiter, + ) + + within_budget: Final = await budget_check( + model="backup", + request_kwargs={"metadata": {"user_api_key_auth": token}}, + llm_router=router, + ) + + assert within_budget is False + assert tuple(call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list) == ( + _member_budget_spend_keys(("backup", "expensive")) + ) + + +@pytest.mark.asyncio +async def test_router_fallback_budget_check_refuses_an_exhausted_alias_cap() -> None: + expensive_model: Final = { + **PAID_MODEL, + "model_name": "expensive", + "model_info": {"id": "expensive-model-id"}, + } + router: Final = Router( + model_list=[FREE_MODEL, expensive_model], + model_group_alias={"backup": "expensive"}, + ) + member_caps: Final = {"backup": {"max_budget": 5.0, "budget_duration": "1d"}} + token: Final = _token(user_id="u1", team_id="t1", team_member_model_max_budget=member_caps) + limiter, redis_cache = await _team_member_limiter(spend_by_model={"backup": 10.0}) + budget_check: Final = RouterFallbackBudgetCheck( + is_enforced=lambda: True, + team_member_model_budget_limiter=limiter, + ) + + within_budget: Final = await budget_check( + model="backup", + request_kwargs={"metadata": {"user_api_key_auth": token}}, + llm_router=router, + ) + + assert within_budget is False + assert tuple(call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list) == ( + _member_budget_spend_keys(("backup",)) + ) + + +@pytest.mark.asyncio +async def test_router_fallback_budget_check_allows_alias_when_both_names_are_within_cap() -> None: + expensive_model: Final = { + **PAID_MODEL, + "model_name": "expensive", + "model_info": {"id": "expensive-model-id"}, + } + router: Final = Router( + model_list=[FREE_MODEL, expensive_model], + model_group_alias={"backup": "expensive"}, + ) + member_caps: Final = { + "backup": {"max_budget": 5.0, "budget_duration": "1d"}, + "expensive": {"max_budget": 5.0, "budget_duration": "1d"}, + } + token: Final = _token(user_id="u1", team_id="t1", team_member_model_max_budget=member_caps) + limiter, redis_cache = await _team_member_limiter(spend_by_model={"backup": 1.0, "expensive": 1.0}) + budget_check: Final = RouterFallbackBudgetCheck( + is_enforced=lambda: True, + team_member_model_budget_limiter=limiter, + ) + + within_budget: Final = await budget_check( + model="backup", + request_kwargs={"metadata": {"user_api_key_auth": token}}, + llm_router=router, + ) + + assert within_budget is True + assert tuple(call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list) == ( + _member_budget_spend_keys(("backup", "expensive")) + ) + + +@pytest.mark.asyncio +async def test_router_fallback_budget_check_deduplicates_non_alias_target() -> None: + router: Final = _router() + member_caps: Final = {"paid-model": {"max_budget": 5.0, "budget_duration": "1d"}} + token: Final = _token(user_id="u1", team_id="t1", team_member_model_max_budget=member_caps) + limiter, redis_cache = await _team_member_limiter(spend_by_model={"paid-model": 1.0}) + budget_check: Final = RouterFallbackBudgetCheck( + is_enforced=lambda: True, + team_member_model_budget_limiter=limiter, + ) + + within_budget: Final = await budget_check( + model="paid-model", + request_kwargs={"metadata": {"user_api_key_auth": token}}, + llm_router=router, + ) + + assert within_budget is True + assert tuple(call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list) == ( + _member_budget_spend_keys(("paid-model",)) + ) + + +@pytest.mark.asyncio +async def test_team_member_budget_allows_a_paid_target_when_under_cap() -> None: + member_caps: Final = {"paid-model": {"max_budget": 5.0, "budget_duration": "1d"}} + token: Final = _token( + user_id="u1", + team_id="t1", + team_member_model_max_budget=member_caps, + ) + limiter, redis_cache = await _team_member_limiter(spend_by_model={"paid-model": 1.0}) + + within_budget: Final = await is_token_within_budget_for_model( + model="paid-model", + valid_token=token, + llm_router=_router(), + team_member_model_budget_limiter=limiter, + ) + + assert within_budget is True + assert tuple(call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list) == ( + _member_budget_spend_keys(("paid-model",)) + ) + + +@pytest.mark.asyncio +async def test_team_member_caps_are_not_evaluated_without_a_limiter() -> None: + token: Final = _token( + user_id="u1", + team_id="t1", + team_member_model_max_budget={"paid-model": {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + + assert await is_token_within_budget_for_model(model="paid-model", valid_token=token, llm_router=_router()) is True + + +@pytest.mark.asyncio +async def test_team_member_limiter_is_not_called_without_member_caps() -> None: + token: Final = _token(user_id="u1", team_id="t1") + limiter, redis_cache = await _team_member_limiter() + + within_budget: Final = await is_token_within_budget_for_model( + model="paid-model", + valid_token=token, + llm_router=_router(), + team_member_model_budget_limiter=limiter, + ) + + assert within_budget is True + assert redis_cache.async_get_cache.call_args_list == [] + + @pytest.mark.asyncio async def test_zero_cost_target_allowed_even_when_over_budget(): """Refusing a free target would deny a request on spend some other model accrued.""" @@ -123,6 +364,33 @@ async def test_enforced_check_reads_the_key_from_request_metadata(metadata_field assert await ENFORCED(model="paid-model", request_kwargs=under, llm_router=_router()) is True +@pytest.mark.asyncio +async def test_router_check_passes_member_limiter_from_request_metadata() -> None: + member_caps: Final = {"paid-model": {"max_budget": 5.0, "budget_duration": "1d"}} + token: Final = _token( + user_id="u1", + team_id="t1", + team_member_model_max_budget=member_caps, + ) + limiter, redis_cache = await _team_member_limiter(spend_by_model={"paid-model": 10.0}) + budget_check: Final = RouterFallbackBudgetCheck( + is_enforced=lambda: True, + team_member_model_budget_limiter=limiter, + ) + request_kwargs: Final = {"metadata": {"user_api_key_auth": token}} + + within_budget: Final = await budget_check( + model="paid-model", + request_kwargs=request_kwargs, + llm_router=_router(), + ) + + assert within_budget is False + assert tuple(call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list) == ( + _member_budget_spend_keys(("paid-model",)) + ) + + @pytest.mark.asyncio async def test_a_stale_low_counter_still_refuses_a_paid_target(monkeypatch): """ diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 1a636e28fb9..492ab408fc8 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -5,10 +5,11 @@ import os import subprocess import sys import time -from collections.abc import Mapping -from contextlib import contextmanager +from collections.abc import Mapping, Sequence +from contextlib import contextmanager, nullcontext from datetime import datetime, timedelta, timezone from functools import partial +from io import BytesIO from pathlib import Path from textwrap import dedent from types import SimpleNamespace @@ -16,13 +17,24 @@ from typing import Final from unittest.mock import ANY, AsyncMock, MagicMock, patch +import orjson import pytest -from fastapi import HTTPException, status +from fastapi import HTTPException, Request, status +from starlette.datastructures import UploadFile import litellm import litellm.proxy.proxy_server from litellm.caching.dual_cache import DualCache -from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.common_utils.user_api_key_cache import ( + NO_TEAM_MEMBERSHIP_SENTINEL, + UserApiKeyCache, + team_membership_reservation_cache_key, +) +from litellm.proxy.hooks.model_max_budget_limiter import ( + PROXY_VirtualKeyModelMaxBudgetLimiter, + model_budget_spend_cache_key, + team_member_budget_entity_id, +) from litellm.proxy._types import ( LiteLLMRoutes, LiteLLM_JWTAuth, @@ -30,6 +42,9 @@ from litellm.proxy._types import ( LiteLLM_EndUserTable, LiteLLM_ObjectPermissionTable, LiteLLM_OrganizationTable, + LiteLLM_ProjectTableCachedObj, + LiteLLM_TeamMembership, + LiteLLM_TeamTable, LiteLLM_TeamTableCachedObj, LiteLLM_UserTable, Litellm_EntityType, @@ -52,8 +67,10 @@ from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.http_parsing_utils import get_client_requested_model from litellm.proxy.auth.user_api_key_auth import ( _check_key_model_budget_with_fallback, + _check_team_member_model_budget_with_fallback, _ensure_litellm_received_at_on_request_state, _ensure_parent_otel_span_on_request_state, + _load_team_member_default_model_budget, _PendingAutoRegister, _matches_routing_override, _reserve_budget_after_common_checks, @@ -78,6 +95,82 @@ class _RoutingRequest: self.state = SimpleNamespace() +async def _real_model_budget_limiter( + *, + member_spend: Mapping[str, float] | None = None, + key_spend: Mapping[str, float] | None = None, + member_errors: Mapping[str, Exception] | None = None, +) -> tuple[PROXY_VirtualKeyModelMaxBudgetLimiter, MagicMock]: + member_entity_id: Final = team_member_budget_entity_id(user_id="user-1", team_id="team-1") + scope_spend: Final = ( + *( + (Litellm_EntityType.TEAM_MEMBER, member_entity_id, model, spend) + for model, spend in (member_spend or {}).items() + ), + *((Litellm_EntityType.KEY, "test-key", model, spend) for model, spend in (key_spend or {}).items()), + ) + scope_errors: Final = tuple( + (Litellm_EntityType.TEAM_MEMBER, member_entity_id, model, error) + for model, error in (member_errors or {}).items() + ) + spend_by_key: Final = { + model_budget_spend_cache_key( + entity_type=entity_type, + entity_id=entity_id, + budget_model=model, + budget_duration="1d", + ): spend + for entity_type, entity_id, model, spend in scope_spend + } + errors_by_key: Final = { + model_budget_spend_cache_key( + entity_type=entity_type, + entity_id=entity_id, + budget_model=model, + budget_duration="1d", + ): error + for entity_type, entity_id, model, error in scope_errors + } + redis_cache: Final = MagicMock() + + async def get_spend(*, key: str) -> object | None: + if key in errors_by_key: + raise errors_by_key[key] + return spend_by_key.get(key) + + redis_cache.async_get_cache = AsyncMock(side_effect=get_spend) + return PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache(redis_cache=redis_cache)), redis_cache + + +def _model_budget_spend_keys( + entity_type: Litellm_EntityType, + entity_id: str, + models: Sequence[str], +) -> list[str]: + return [ + model_budget_spend_cache_key( + entity_type=entity_type, + entity_id=entity_id, + budget_model=model, + budget_duration="1d", + ) + for model in models + ] + + +def _team_member_model_budget_spend_keys(models: Sequence[str]) -> list[str]: + return _model_budget_spend_keys( + Litellm_EntityType.TEAM_MEMBER, + team_member_budget_entity_id(user_id="user-1", team_id="team-1"), + models, + ) + + +def _assert_budget_exceeded_values(error: litellm.BudgetExceededError) -> None: + assert error.current_cost == 10 + assert error.max_budget == 5 + + def test_get_api_key(): bearer_token = "Bearer sk-98765678" api_key = "sk-98765678" @@ -4730,9 +4823,11 @@ async def test_centralized_common_checks_carries_team_and_user_budget_state_on_t ) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj.async_service_success_hook = AsyncMock() + database = MagicMock() + database.db.litellm_teammembership.find_unique = AsyncMock(return_value=None) attrs = { **_proxy_attrs_for_centralized_checks(user_custom_auth=None), - "prisma_client": MagicMock(), + "prisma_client": database, "user_api_key_cache": user_api_key_cache, "proxy_logging_obj": proxy_logging_obj, } @@ -5011,6 +5106,503 @@ async def test_centralized_common_checks_enforces_team_model_max_budget_from_the assert limiter.calls == [("t1", team_caps, key_caps, "gpt-4o")] +@pytest.mark.asyncio +async def test_centralized_common_checks_loads_member_budget_and_exempts_discovery_routes(): + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.hooks.model_max_budget_limiter import PROXY_VirtualKeyModelMaxBudgetLimiter + import litellm.proxy.proxy_server as _proxy_server_mod + + budget_id: Final = "member-budget-1" + member_caps: Final = {"gpt-4o": {"max_budget": 0.0, "budget_duration": "1d"}} + membership: Final = LiteLLM_TeamMembership( + user_id="u1", + team_id="t1", + budget_id=budget_id, + litellm_budget_table=LiteLLM_BudgetTable( + budget_id=budget_id, + model_max_budget={"gpt-4o": {"max_budget": 100.0, "budget_duration": "1d"}}, + ), + ) + user_api_key_cache: Final = UserApiKeyCache() + await user_api_key_cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id="u1", team_id="t1"), + value=membership, + model_type=LiteLLM_TeamMembership, + ) + await user_api_key_cache.async_set_cache( + key="team_id:t1", + value=LiteLLM_TeamTableCachedObj( + team_id="t1", + models=["gpt-4o"], + metadata={"team_member_budget_id": budget_id}, + ), + model_type=LiteLLM_TeamTableCachedObj, + ) + await user_api_key_cache.async_set_cache( + key=f"team_member_default_budget:{budget_id}", + value=LiteLLM_BudgetTable(budget_id=budget_id, model_max_budget=member_caps), + model_type=LiteLLM_BudgetTable, + ) + await user_api_key_cache.async_set_cache( + key="u1", + value=LiteLLM_UserTable(user_id="u1", user_role=LitellmUserRoles.INTERNAL_USER), + model_type=LiteLLM_UserTable, + ) + router: Final = litellm.Router( + model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}}], + ) + limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache()) + attrs: Final = { + "general_settings": {"disable_budget_reservation": True}, + "litellm_proxy_admin_name": "admin", + "llm_model_list": router.model_list, + "llm_router": router, + "master_key": "sk-master-key", + "prisma_client": MagicMock(), + "user_api_key_cache": user_api_key_cache, + "proxy_logging_obj": MagicMock(), + "model_max_budget_limiter": limiter, + "user_custom_auth": None, + } + originals: Final = {attribute: getattr(_proxy_server_mod, attribute, None) for attribute in attrs} + + try: + for attribute, value in attrs.items(): + setattr(_proxy_server_mod, attribute, value) + token: Final = UserAPIKeyAuth( + api_key="sk-test", + token="hashed", + user_id="u1", + team_id="t1", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + request: Final = Request( + scope={ + "type": "http", + "method": "POST", + "path": "/v1/models", + "raw_path": b"/v1/models", + "query_string": b"", + "headers": [], + "server": ("test", 80), + "client": ("test", 1), + "scheme": "http", + } + ) + await run_centralized_common_checks( + user_api_key_auth_obj=token, + request=request, + request_data={"model": "gpt-4o"}, + route="/v1/models", + ) + + assert token.team_member_model_max_budget == member_caps + + non_exempt_token: Final = UserAPIKeyAuth( + api_key="sk-test", + token="hashed", + user_id="u1", + team_id="t1", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + non_exempt_request: Final = Request( + scope={ + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "raw_path": b"/v1/chat/completions", + "query_string": b"", + "headers": [], + "server": ("test", 80), + "client": ("test", 1), + "scheme": "http", + } + ) + with pytest.raises(litellm.BudgetExceededError): + await run_centralized_common_checks( + user_api_key_auth_obj=non_exempt_token, + request=non_exempt_request, + request_data={"model": "gpt-4o"}, + route="/v1/chat/completions", + ) + finally: + for attribute, value in originals.items(): + setattr(_proxy_server_mod, attribute, value) + + +@pytest.mark.parametrize( + ("route", "path", "must_raise"), + [ + pytest.param("/v1/models", "/v1/models", False, id="budget-exempt-discovery-route"), + pytest.param("/chat/completions", "/v1/chat/completions", True, id="paid-model-route"), + ], +) +@pytest.mark.asyncio +async def test_centralized_common_checks_skips_default_member_budget_lookup_errors_on_exempt_routes( + route: str, + path: str, + must_raise: bool, +): + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.hooks.model_max_budget_limiter import PROXY_VirtualKeyModelMaxBudgetLimiter + import litellm.proxy.proxy_server as _proxy_server_mod + + budget_id: Final = "member-budget-lookup-fails" + membership: Final = LiteLLM_TeamMembership( + user_id="u1", + team_id="t1", + budget_id=budget_id, + litellm_budget_table=LiteLLM_BudgetTable( + budget_id=budget_id, + model_max_budget={"gpt-4o": {"max_budget": 100.0, "budget_duration": "1d"}}, + ), + ) + user_api_key_cache: Final = UserApiKeyCache() + await user_api_key_cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id="u1", team_id="t1"), + value=membership, + model_type=LiteLLM_TeamMembership, + ) + await user_api_key_cache.async_set_cache( + key="team_id:t1", + value=LiteLLM_TeamTableCachedObj( + team_id="t1", + models=["gpt-4o"], + metadata={"team_member_budget_id": budget_id}, + ), + model_type=LiteLLM_TeamTableCachedObj, + ) + await user_api_key_cache.async_set_cache( + key="u1", + value=LiteLLM_UserTable(user_id="u1", user_role=LitellmUserRoles.INTERNAL_USER), + model_type=LiteLLM_UserTable, + ) + prisma_client: Final = MagicMock() + prisma_client.db.litellm_budgettable.find_unique = AsyncMock( + side_effect=RuntimeError("budget database unavailable") + ) + router: Final = litellm.Router( + model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}}], + ) + limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache()) + attrs: Final = { + "general_settings": {"disable_budget_reservation": True}, + "litellm_proxy_admin_name": "admin", + "llm_model_list": router.model_list, + "llm_router": router, + "master_key": "sk-master-key", + "prisma_client": prisma_client, + "user_api_key_cache": user_api_key_cache, + "proxy_logging_obj": MagicMock(), + "model_max_budget_limiter": limiter, + "user_custom_auth": None, + } + originals: Final = {attribute: getattr(_proxy_server_mod, attribute, None) for attribute in attrs} + + try: + for attribute, value in attrs.items(): + setattr(_proxy_server_mod, attribute, value) + token: Final = UserAPIKeyAuth( + api_key="sk-test", + token="hashed", + user_id="u1", + team_id="t1", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + request: Final = Request( + scope={ + "type": "http", + "method": "POST", + "path": path, + "raw_path": path.encode(), + "query_string": b"", + "headers": [], + "server": ("test", 80), + "client": ("test", 1), + "scheme": "http", + } + ) + with pytest.raises(RuntimeError, match="budget database unavailable") if must_raise else nullcontext(): + await run_centralized_common_checks( + user_api_key_auth_obj=token, + request=request, + request_data={"model": "gpt-4o"}, + route=route, + ) + + assert token.team_member_model_max_budget is None + prisma_client.db.litellm_budgettable.find_unique.assert_awaited_once_with(where={"budget_id": budget_id}) + finally: + for attribute, value in originals.items(): + setattr(_proxy_server_mod, attribute, value) + + +def _team_member_model_budget_load_inputs() -> tuple[ + UserAPIKeyAuth, LiteLLM_TeamTableCachedObj, MagicMock, UserApiKeyCache +]: + token: Final = UserAPIKeyAuth(user_id="member-1", team_id="team-1") + team_object: Final = LiteLLM_TeamTableCachedObj( + team_id="team-1", + metadata={"team_member_budget_id": "default-budget"}, + ) + prisma_client: Final = MagicMock() + prisma_client.db.litellm_budgettable.find_unique = AsyncMock() + user_api_key_cache: Final = UserApiKeyCache() + return token, team_object, prisma_client, user_api_key_cache + + +@pytest.mark.asyncio +async def test_member_model_budget_load_prefers_override_row() -> None: + token, team_object, prisma_client, user_api_key_cache = _team_member_model_budget_load_inputs() + member_caps: Final = {"claude-sonnet-4-6": {"max_budget": 2.0, "budget_duration": "1d"}} + membership: Final = LiteLLM_TeamMembership( + user_id="member-1", + team_id="team-1", + budget_id="member-budget", + litellm_budget_table=LiteLLM_BudgetTable(budget_id="member-budget", model_max_budget=member_caps), + ) + await user_api_key_cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id="member-1", team_id="team-1"), + value=membership, + model_type=LiteLLM_TeamMembership, + ) + + await _load_team_member_default_model_budget( + user_api_key_auth_obj=token, + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + skip_budget_checks=False, + proxy_logging_obj=None, + ) + + assert token.team_member_model_max_budget == member_caps + prisma_client.db.litellm_budgettable.find_unique.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_member_model_budget_load_uses_override_without_team_default_id() -> None: + token, team_object, prisma_client, user_api_key_cache = _team_member_model_budget_load_inputs() + team_object = LiteLLM_TeamTableCachedObj(team_id=team_object.team_id, metadata={}) + member_caps: Final = {"claude-sonnet-4-6": {"max_budget": 2.0, "budget_duration": "1d"}} + membership: Final = LiteLLM_TeamMembership( + user_id="member-1", + team_id="team-1", + budget_id="member-budget", + litellm_budget_table=LiteLLM_BudgetTable(budget_id="member-budget", model_max_budget=member_caps), + ) + await user_api_key_cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id="member-1", team_id="team-1"), + value=membership, + model_type=LiteLLM_TeamMembership, + ) + + await _load_team_member_default_model_budget( + user_api_key_auth_obj=token, + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + skip_budget_checks=False, + proxy_logging_obj=None, + ) + + assert token.team_member_model_max_budget == member_caps + prisma_client.db.litellm_budgettable.find_unique.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_member_model_budget_load_reads_current_default_for_plain_team_object() -> None: + from litellm.proxy.common_utils.user_api_key_cache import team_membership_reservation_cache_key + + token: Final = UserAPIKeyAuth(user_id="member-1", team_id="team-1") + team_object: Final = LiteLLM_TeamTable( + team_id="team-1", + metadata={"team_member_budget_id": "default-budget"}, + ) + prisma_client: Final = MagicMock() + user_api_key_cache: Final = UserApiKeyCache() + stale_caps: Final = {"claude-sonnet-4-6": {"max_budget": 3.0, "budget_duration": "1d"}} + current_caps: Final = {"claude-sonnet-4-6": {"max_budget": 1.0, "budget_duration": "1d"}} + membership: Final = LiteLLM_TeamMembership( + user_id="member-1", + team_id="team-1", + budget_id="default-budget", + litellm_budget_table=LiteLLM_BudgetTable(budget_id="default-budget", model_max_budget=stale_caps), + ) + await user_api_key_cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id="member-1", team_id="team-1"), + value=membership, + model_type=LiteLLM_TeamMembership, + ) + await user_api_key_cache.async_set_cache( + key="team_member_default_budget:default-budget", + value=LiteLLM_BudgetTable(budget_id="default-budget", model_max_budget=current_caps), + model_type=LiteLLM_BudgetTable, + ) + + await _load_team_member_default_model_budget( + user_api_key_auth_obj=token, + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + skip_budget_checks=False, + proxy_logging_obj=None, + ) + + assert token.team_member_model_max_budget == current_caps + + +@pytest.mark.asyncio +async def test_member_model_budget_load_overall_only_override_keeps_default_caps() -> None: + token, team_object, prisma_client, user_api_key_cache = _team_member_model_budget_load_inputs() + default_caps: Final = {"claude-sonnet-4-6": {"max_budget": 3.0, "budget_duration": "1d"}} + membership: Final = LiteLLM_TeamMembership( + user_id="member-1", + team_id="team-1", + budget_id="member-budget", + litellm_budget_table=LiteLLM_BudgetTable(budget_id="member-budget", max_budget=5.0, model_max_budget=None), + ) + await user_api_key_cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id="member-1", team_id="team-1"), + value=membership, + model_type=LiteLLM_TeamMembership, + ) + await user_api_key_cache.async_set_cache( + key="team_member_default_budget:default-budget", + value=LiteLLM_BudgetTable(budget_id="default-budget", model_max_budget=default_caps), + model_type=LiteLLM_BudgetTable, + ) + + await _load_team_member_default_model_budget( + user_api_key_auth_obj=token, + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + skip_budget_checks=False, + proxy_logging_obj=None, + ) + + assert token.team_member_model_max_budget == default_caps + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cleared_member_caps", [None, {}], ids=["null", "empty"]) +async def test_member_model_budget_load_falls_back_when_override_has_no_model_caps( + cleared_member_caps: dict[str, object] | None, +) -> None: + token, team_object, prisma_client, user_api_key_cache = _team_member_model_budget_load_inputs() + default_caps: Final = {"claude-sonnet-4-6": {"max_budget": 3.0, "budget_duration": "1d"}} + membership: Final = LiteLLM_TeamMembership( + user_id="member-1", + team_id="team-1", + budget_id="member-budget", + litellm_budget_table=LiteLLM_BudgetTable(budget_id="member-budget", model_max_budget=cleared_member_caps), + ) + default_budget: Final = LiteLLM_BudgetTable(budget_id="default-budget", model_max_budget=default_caps) + await user_api_key_cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id="member-1", team_id="team-1"), + value=membership, + model_type=LiteLLM_TeamMembership, + ) + await user_api_key_cache.async_set_cache( + key="team_member_default_budget:default-budget", + value=default_budget, + model_type=LiteLLM_BudgetTable, + ) + + await _load_team_member_default_model_budget( + user_api_key_auth_obj=token, + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + skip_budget_checks=False, + proxy_logging_obj=None, + ) + + assert token.team_member_model_max_budget == default_caps + + +@pytest.mark.asyncio +async def test_member_model_budget_load_uses_default_without_membership() -> None: + token, team_object, prisma_client, user_api_key_cache = _team_member_model_budget_load_inputs() + default_caps: Final = {"claude-sonnet-4-6": {"max_budget": 3.0, "budget_duration": "1d"}} + default_budget: Final = LiteLLM_BudgetTable(budget_id="default-budget", model_max_budget=default_caps) + prisma_client.db.litellm_teammembership.find_unique = AsyncMock(return_value=None) + await user_api_key_cache.async_set_cache( + key="team_member_default_budget:default-budget", + value=default_budget, + model_type=LiteLLM_BudgetTable, + ) + + await _load_team_member_default_model_budget( + user_api_key_auth_obj=token, + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + skip_budget_checks=False, + proxy_logging_obj=None, + ) + + assert token.team_member_model_max_budget == default_caps + + +@pytest.mark.asyncio +async def test_member_model_budget_load_fails_closed_on_default_lookup_error() -> None: + token, team_object, prisma_client, user_api_key_cache = _team_member_model_budget_load_inputs() + prisma_client.db.litellm_teammembership.find_unique = AsyncMock(return_value=None) + prisma_client.db.litellm_budgettable.find_unique = AsyncMock( + side_effect=RuntimeError("budget database unavailable") + ) + + with pytest.raises(RuntimeError, match="budget database unavailable"): + await _load_team_member_default_model_budget( + user_api_key_auth_obj=token, + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + skip_budget_checks=False, + proxy_logging_obj=None, + ) + + +@pytest.mark.asyncio +async def test_member_model_budget_load_skips_default_lookup_error_when_checks_are_skipped() -> None: + token, team_object, prisma_client, user_api_key_cache = _team_member_model_budget_load_inputs() + prisma_client.db.litellm_teammembership.find_unique = AsyncMock(return_value=None) + prisma_client.db.litellm_budgettable.find_unique = AsyncMock( + side_effect=RuntimeError("budget database unavailable") + ) + + await _load_team_member_default_model_budget( + user_api_key_auth_obj=token, + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + skip_budget_checks=True, + proxy_logging_obj=None, + ) + + assert token.team_member_model_max_budget is None + + +@pytest.mark.asyncio +async def test_member_model_budget_load_allows_missing_default_row() -> None: + token, team_object, prisma_client, user_api_key_cache = _team_member_model_budget_load_inputs() + prisma_client.db.litellm_teammembership.find_unique = AsyncMock(return_value=None) + prisma_client.db.litellm_budgettable.find_unique = AsyncMock(return_value=None) + + await _load_team_member_default_model_budget( + user_api_key_auth_obj=token, + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + skip_budget_checks=False, + proxy_logging_obj=None, + ) + + assert token.team_member_model_max_budget is None + + @pytest.mark.asyncio async def test_centralized_common_checks_skipped_for_custom_auth_without_flag(): """Existing RPS guarantee: custom-auth deployments without @@ -5331,11 +5923,13 @@ async def test_team_key_vector_store_access_when_team_cannot_be_resolved( request._url = URL(url="/v1/rag/query") database: Final = MagicMock() database.db.litellm_objectpermissiontable.find_unique = AsyncMock( - side_effect=lambda where: LiteLLM_ObjectPermissionTable( - object_permission_id=where["object_permission_id"], vector_stores=["KBSTOREA"] + side_effect=lambda where: ( + LiteLLM_ObjectPermissionTable( + object_permission_id=where["object_permission_id"], vector_stores=["KBSTOREA"] + ) + if where["object_permission_id"] == "key-permission" + else None ) - if where["object_permission_id"] == "key-permission" - else None ) attrs: Final = { **_proxy_attrs_for_centralized_checks(), @@ -5387,6 +5981,7 @@ async def test_keyless_team_member_vector_store_access_uses_only_the_resolved_te ): from fastapi import Request from starlette.datastructures import URL + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache token: Final = UserAPIKeyAuth( user_id="user-1", @@ -5408,9 +6003,27 @@ async def test_keyless_team_member_vector_store_access_uses_only_the_resolved_te object_permission_id=where["object_permission_id"], vector_stores=grants[where["object_permission_id"]] ) ) + database.db.litellm_teammembership.find_unique = AsyncMock(return_value=None) + if isinstance(team_lookup, Exception): + database.db.litellm_teamtable.find_unique = AsyncMock(side_effect=team_lookup) + else: + database.db.litellm_teamtable.find_unique = AsyncMock( + side_effect=lambda where, **_: LiteLLM_TeamTable( + team_id=where["team_id"], + models=["gpt-4o-mini"], + object_permission_id=team_lookup[where["team_id"]], + ) + ) + user_api_key_cache: Final = UserApiKeyCache() + await user_api_key_cache.async_set_cache( + key="user-1", + value=LiteLLM_UserTable(user_id="user-1", teams=["team-1", "team-2"], object_permission_id="user-grants-a"), + model_type=LiteLLM_UserTable, + ) attrs: Final = { **_proxy_attrs_for_centralized_checks(), "prisma_client": database, + "user_api_key_cache": user_api_key_cache, "proxy_logging_obj": MagicMock(service_logging_obj=MagicMock(async_service_success_hook=AsyncMock())), "general_settings": {"vector_store_deny_by_default": True}, } @@ -5418,22 +6031,6 @@ async def test_keyless_team_member_vector_store_access_uses_only_the_resolved_te monkeypatch.setattr(litellm.proxy.proxy_server, name, value) monkeypatch.setattr(litellm, "vector_store_registry", None) - async def get_team(team_id: str, **_: object) -> LiteLLM_TeamTableCachedObj: - if isinstance(team_lookup, Exception): - raise team_lookup - return LiteLLM_TeamTableCachedObj(team_id=team_id, models=["gpt-4o-mini"], object_permission_id=team_lookup[team_id]) - - monkeypatch.setattr("litellm.proxy.auth.user_api_key_auth.get_team_object", get_team) - monkeypatch.setattr("litellm.proxy.auth.auth_checks.get_team_membership", AsyncMock(return_value=None)) - monkeypatch.setattr( - "litellm.proxy.auth.user_api_key_auth.get_user_object", - AsyncMock( - return_value=LiteLLM_UserTable( - user_id="user-1", teams=["team-1", "team-2"], object_permission_id="user-grants-a" - ) - ), - ) - checks: Final = run_centralized_common_checks( user_api_key_auth_obj=token, request=request, @@ -6683,6 +7280,7 @@ async def test_centralized_common_checks_inherits_org_identity( attrs = _proxy_attrs_for_centralized_checks(user_custom_auth=None) attrs["prisma_client"] = MagicMock() + attrs["prisma_client"].db.litellm_teammembership.find_unique = AsyncMock(return_value=None) attrs["general_settings"] = {"allow_requests_on_db_unavailable": allow_db_unavailable} originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs} try: @@ -7690,7 +8288,8 @@ async def _authenticate_session_token_against_db( from starlette.datastructures import URL attrs = _proxy_attrs_for_db_lookup() - attrs["prisma_client"].db.litellm_teammembership.find_first = AsyncMock(return_value=None) + attrs["prisma_client"].db.litellm_teammembership.find_unique = AsyncMock(return_value=membership_row) + attrs["prisma_client"].db.litellm_teammembership.find_first = AsyncMock(return_value=membership_row) originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs} assemble = AsyncMock(return_value=UserAPIKeyAuth(user_id=SESSION_USER_ID, is_session_token=True)) try: @@ -7710,10 +8309,6 @@ async def _authenticate_session_token_against_db( "litellm.proxy.auth.user_api_key_auth.get_team_object", AsyncMock(return_value=team_row, side_effect=team_error), ), - patch( # test-quality-ok: the builder reads its DB row loaders off module globals - "litellm.proxy.auth.user_api_key_auth.get_team_membership", - AsyncMock(return_value=membership_row), - ), ): await user_api_key_auth_builder( request=request, @@ -8226,25 +8821,16 @@ class TestCheckKeyModelBudgetWithFallback: request_data = {"model": "gpt-4o"} request = self._make_request() - with patch( - "litellm.proxy.auth.user_api_key_auth.can_key_call_model", - side_effect=ProxyException( - message="model not allowed", - type=ProxyErrorTypes.budget_exceeded, - param="model", - code=status.HTTP_403_FORBIDDEN, - ), - ): - with pytest.raises(litellm.BudgetExceededError) as exc_info: - await _check_key_model_budget_with_fallback( - valid_token=valid_token, - model_max_budget_limiter=limiter, - model_name="gpt-4o", - request_data=request_data, - request=request, - llm_model_list=None, - llm_router=None, - ) + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_key_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + model_name="gpt-4o", + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=None, + ) assert exc_info.value is original_error assert request_data["model"] == "gpt-4o" @@ -8264,19 +8850,15 @@ class TestCheckKeyModelBudgetWithFallback: request_data = {"model": "gpt-4o"} request = self._make_request() - with patch( - "litellm.proxy.auth.user_api_key_auth.can_key_call_model", - return_value=True, - ): - await _check_key_model_budget_with_fallback( - valid_token=valid_token, - model_max_budget_limiter=limiter, - model_name="gpt-4o", - request_data=request_data, - request=request, - llm_model_list=None, - llm_router=None, - ) + await _check_key_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + model_name="gpt-4o", + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=None, + ) assert request_data["model"] == "gpt-4o-mini" @@ -8298,20 +8880,16 @@ class TestCheckKeyModelBudgetWithFallback: request_data = {"model": "gpt-4o"} request = self._make_request() - with patch( - "litellm.proxy.auth.user_api_key_auth.can_key_call_model", - return_value=True, - ): - with pytest.raises(litellm.BudgetExceededError) as exc_info: - await _check_key_model_budget_with_fallback( - valid_token=valid_token, - model_max_budget_limiter=limiter, - model_name="gpt-4o", - request_data=request_data, - request=request, - llm_model_list=None, - llm_router=None, - ) + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_key_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + model_name="gpt-4o", + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=None, + ) assert exc_info.value is original_error assert request_data["model"] == "gpt-4o" @@ -8333,19 +8911,15 @@ class TestCheckKeyModelBudgetWithFallback: request = self._make_request() request.scope["path_params"] = {"model": "gpt-4o"} - with patch( - "litellm.proxy.auth.user_api_key_auth.can_key_call_model", - return_value=True, - ): - await _check_key_model_budget_with_fallback( - valid_token=valid_token, - model_max_budget_limiter=limiter, - model_name="gpt-4o", - request_data=request_data, - request=request, - llm_model_list=None, - llm_router=None, - ) + await _check_key_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + model_name="gpt-4o", + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=None, + ) assert request_data["model"] == "gpt-4o-mini" assert request.scope["path_params"]["model"] == "gpt-4o-mini" @@ -8379,6 +8953,1287 @@ class TestCheckKeyModelBudgetWithFallback: assert "model" not in request_data +class TestCheckTeamMemberModelBudgetWithFallback: + @pytest.fixture(autouse=True) + async def _seed_auth_cache(self, monkeypatch: pytest.MonkeyPatch) -> None: + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, team_membership_reservation_cache_key + + user_api_key_cache: Final = UserApiKeyCache() + await user_api_key_cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id="user-1", team_id="team-1"), + value=NO_TEAM_MEMBERSHIP_SENTINEL, + ) + await user_api_key_cache.async_set_cache( + key="team_id:team-1", + value=LiteLLM_TeamTableCachedObj(team_id="team-1"), + model_type=LiteLLM_TeamTableCachedObj, + ) + prisma_client: Final = MagicMock() + prisma_client.db.litellm_teammembership.find_unique = AsyncMock(return_value=None) + prisma_client.db.litellm_teammembership.find_first = AsyncMock(return_value=None) + monkeypatch.setattr(proxy_server, "user_api_key_cache", user_api_key_cache) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock()) + + def _router( + self, + fallbacks: list[dict[str, object]] | None, + model_group_alias: dict[str, str] | None = None, + free_models: frozenset[str] = frozenset(), + ) -> litellm.Router: + model_list: Final = [ + { + "model_name": model, + "litellm_params": {"model": "gpt-4o"}, + **( + {"model_info": {"input_cost_per_token": 0, "output_cost_per_token": 0}} + if model in free_models + else {} + ), + } + for model in ( + "gpt-4o", + "gpt-4o-mini", + "restricted-model", + "backup", + "backup-unavailable", + "backup-valid", + "claude-sonnet-4-6", + "claude-haiku-4-5", + "claude-haiku", + "expensive", + *sorted(free_models), + ) + ] + return litellm.Router(model_list=model_list, fallbacks=fallbacks, model_group_alias=model_group_alias) + + def _request(self, model: str) -> Request: + return Request(scope={"type": "http", "path_params": {"model": model}, "headers": []}) + + @pytest.mark.asyncio + async def test_within_budget_does_not_rewrite_request(self): + budget: Final = {"gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}} + valid_token: Final = UserAPIKeyAuth( + token="test-key", + user_id="user-1", + team_id="team-1", + team_member_model_max_budget=budget, + ) + limiter, _ = await _real_model_budget_limiter() + request_data: Final = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"gpt-4o": ["gpt-4o-mini"]}]), + ) + + assert request_data["model"] == "gpt-4o" + assert request.scope["path_params"]["model"] == "gpt-4o" + assert "parsed_body" not in request.scope + + @pytest.mark.asyncio + async def test_generic_fallback_rewrites_every_request_view(self): + budget: Final = {"gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}} + valid_token: Final = UserAPIKeyAuth( + token="test-key", + models=["gpt-4o", "gpt-4o-mini"], + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={**budget, "gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}}, + ) + limiter, _ = await _real_model_budget_limiter(member_spend={"gpt-4o": 10.0}) + request_data: Final = {"model": "gpt-4o", "auth_injected_metadata": {"internal": "value"}} + request: Final = self._request("gpt-4o") + client_body: Final = {"model": "gpt-4o", "messages": [{"role": "user", "content": "hello"}]} + request.scope["parsed_body"] = (tuple(client_body), client_body) + + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"*": ["gpt-4o-mini"]}]), + ) + + assert request_data["model"] == "gpt-4o-mini" + expected_body: Final = {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hello"}]} + assert request._json == expected_body + assert orjson.loads(request._body) == expected_body + assert request.scope["parsed_body"][1] == expected_body + assert "auth_injected_metadata" not in request._json + assert request.scope["path_params"]["model"] == "gpt-4o-mini" + + @pytest.mark.asyncio + async def test_specific_fallback_rewrites_requested_model(self): + valid_token: Final = UserAPIKeyAuth( + token="test-key", + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={ + "gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}, + "gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + limiter, _ = await _real_model_budget_limiter(member_spend={"gpt-4o": 10.0}) + request_data: Final = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"gpt-4o": ["gpt-4o-mini"]}]), + ) + + assert request_data["model"] == "gpt-4o-mini" + assert request.scope["path_params"]["model"] == "gpt-4o-mini" + + @pytest.mark.asyncio + async def test_multipart_request_keeps_member_budget_error_without_rewriting(self) -> None: + valid_token: Final = UserAPIKeyAuth( + token="test-key", + models=["claude-sonnet-4-6", "claude-haiku-4-5"], + user_id="user-1", + team_id="team-1", + router_settings={"fallbacks": [{"claude-sonnet-4-6": ["claude-haiku-4-5"]}]}, + team_member_model_max_budget={"claude-sonnet-4-6": {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, redis_cache = await _real_model_budget_limiter(member_spend={"claude-sonnet-4-6": 10.0}) + user_api_key_cache: Final = UserApiKeyCache() + await user_api_key_cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id="user-1", team_id="team-1"), + value=NO_TEAM_MEMBERSHIP_SENTINEL, + ) + uploaded_file: Final = UploadFile(filename="audio.wav", file=BytesIO(b"audio")) + request_data: Final = {"model": "claude-sonnet-4-6", "file": uploaded_file} + request: Final = Request( + scope={ + "type": "http", + "path_params": {"model": "claude-sonnet-4-6"}, + "headers": [(b"content-type", b"Multipart/Form-Data; boundary=test")], + } + ) + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["claude-sonnet-4-6"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=litellm.Router( + model_list=[], + fallbacks=[{"claude-sonnet-4-6": ["claude-haiku-4-5"]}], + ), + user_api_key_cache=user_api_key_cache, + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == "claude-sonnet-4-6" + assert request_data["file"] is uploaded_file + assert request.scope["path_params"]["model"] == "claude-sonnet-4-6" + assert getattr(request, "_body", None) is None + assert [ + call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list + ] == _team_member_model_budget_spend_keys(["claude-sonnet-4-6"]) + + @pytest.mark.asyncio + async def test_provider_scoped_resource_keeps_member_budget_error_without_fallback(self) -> None: + valid_token: Final = UserAPIKeyAuth( + token="test-key", + models=["claude-sonnet-4-6", "claude-haiku-4-5"], + user_id="user-1", + team_id="team-1", + router_settings={"fallbacks": [{"claude-sonnet-4-6": ["claude-haiku-4-5"]}]}, + team_member_model_max_budget={"claude-sonnet-4-6": {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, redis_cache = await _real_model_budget_limiter(member_spend={"claude-sonnet-4-6": 10.0}) + request_data: Final[dict[str, object]] = { + "model": "claude-sonnet-4-6", + "input_file_id": "file-123", + } + request: Final = self._request("claude-sonnet-4-6") + user_api_key_cache: Final = UserApiKeyCache() + await user_api_key_cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id="user-1", team_id="team-1"), + value=NO_TEAM_MEMBERSHIP_SENTINEL, + ) + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["claude-sonnet-4-6"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=litellm.Router( + model_list=[], + fallbacks=[{"claude-sonnet-4-6": ["claude-haiku-4-5"]}], + ), + user_api_key_cache=user_api_key_cache, + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data == { + "model": "claude-sonnet-4-6", + "input_file_id": "file-123", + } + assert request.scope["path_params"]["model"] == "claude-sonnet-4-6" + assert [ + call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list + ] == _team_member_model_budget_spend_keys(["claude-sonnet-4-6"]) + + @pytest.mark.asyncio + async def test_fallback_prefers_hierarchical_router_settings_fallbacks(self) -> None: + import litellm.proxy.proxy_server as proxy_server + + router_settings: Final = {"fallbacks": [{"claude-sonnet-4-6": ["claude-haiku-4-5"]}]} + await proxy_server.user_api_key_cache.async_set_cache( + key="team_id:team-1", + value=LiteLLM_TeamTableCachedObj(team_id="team-1", router_settings=router_settings), + model_type=LiteLLM_TeamTableCachedObj, + ) + valid_token: Final = UserAPIKeyAuth( + token="test-key", + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={ + "claude-sonnet-4-6": {"max_budget": 5.0, "budget_duration": "1d"}, + "claude-haiku-4-5": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + limiter, redis_cache = await _real_model_budget_limiter(member_spend={"claude-sonnet-4-6": 10.0}) + request_data: Final = {"model": "claude-sonnet-4-6"} + request: Final = self._request("claude-sonnet-4-6") + router: Final = self._router([{"claude-sonnet-4-6": ["global-backup"]}]) + + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["claude-sonnet-4-6"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=router, + ) + + assert request_data["model"] == "claude-haiku-4-5" + assert request.scope["path_params"]["model"] == "claude-haiku-4-5" + assert [ + call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list + ] == _team_member_model_budget_spend_keys(["claude-sonnet-4-6", "claude-haiku-4-5"]) + + @pytest.mark.asyncio + async def test_fallback_resolves_router_alias_before_budget_check(self) -> None: + valid_token: Final = UserAPIKeyAuth( + token="test-key", + user_id="user-1", + team_id="team-1", + router_settings={"model_group_alias": {"backup": "expensive"}}, + team_member_model_max_budget={ + "claude-sonnet-4-6": {"max_budget": 5.0, "budget_duration": "1d"}, + "backup": {"max_budget": 100.0, "budget_duration": "1d"}, + "expensive": {"max_budget": 5.0, "budget_duration": "1d"}, + }, + ) + limiter, redis_cache = await _real_model_budget_limiter( + member_spend={"claude-sonnet-4-6": 10.0, "expensive": 10.0} + ) + request_data: Final = {"model": "claude-sonnet-4-6"} + request: Final = self._request("claude-sonnet-4-6") + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["claude-sonnet-4-6"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"claude-sonnet-4-6": ["backup"]}]), + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == "claude-sonnet-4-6" + assert [ + call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list + ] == _team_member_model_budget_spend_keys(["claude-sonnet-4-6", "backup", "expensive"]) + + @pytest.mark.asyncio + async def test_fallback_checks_global_router_alias_budget_before_rerouting(self) -> None: + valid_token: Final = UserAPIKeyAuth( + token="test-key", + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={ + "claude-sonnet-4-6": {"max_budget": 5.0, "budget_duration": "1d"}, + "backup": {"max_budget": 100.0, "budget_duration": "1d"}, + "expensive": {"max_budget": 5.0, "budget_duration": "1d"}, + }, + ) + limiter, redis_cache = await _real_model_budget_limiter( + member_spend={"claude-sonnet-4-6": 10.0, "expensive": 10.0} + ) + request_data: Final[dict[str, object]] = {"model": "claude-sonnet-4-6"} + request: Final = self._request("claude-sonnet-4-6") + router: Final = self._router( + [{"claude-sonnet-4-6": ["backup"]}], + model_group_alias={"backup": "expensive"}, + ) + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["claude-sonnet-4-6"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=router, + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == "claude-sonnet-4-6" + assert [ + call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list + ] == _team_member_model_budget_spend_keys(["claude-sonnet-4-6", "backup", "expensive"]) + + @pytest.mark.asyncio + async def test_fallback_checks_key_team_alias_budget_before_rerouting(self) -> None: + model_group_alias: Final = {"backup": "expensive"} + valid_token: Final = UserAPIKeyAuth( + token="test-key", + user_id="user-1", + team_id="team-1", + router_settings={"model_group_alias": model_group_alias}, + team_member_model_max_budget={ + "claude-sonnet-4-6": {"max_budget": 5.0, "budget_duration": "1d"}, + "backup": {"max_budget": 5.0, "budget_duration": "1d"}, + }, + ) + limiter, redis_cache = await _real_model_budget_limiter( + member_spend={"claude-sonnet-4-6": 10.0, "backup": 10.0} + ) + request_data: Final[dict[str, object]] = {"model": "claude-sonnet-4-6"} + request: Final = self._request("claude-sonnet-4-6") + router: Final = self._router([{"claude-sonnet-4-6": ["backup"]}]) + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["claude-sonnet-4-6"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=router, + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == "claude-sonnet-4-6" + assert [ + call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list + ] == _team_member_model_budget_spend_keys(["claude-sonnet-4-6", "backup"]) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "key_aliases, global_alias_map", + [ + pytest.param({"backup": "expensive"}, {}, id="key_aliases"), + pytest.param({}, {"backup": "expensive"}, id="global_alias_map"), + ], + ) + async def test_fallback_aliased_to_spent_member_model_budget_is_skipped( + self, + monkeypatch: pytest.MonkeyPatch, + key_aliases: dict[str, str], + global_alias_map: dict[str, str], + ) -> None: + primary_model: Final = "claude-sonnet-4-6" + fallback_model: Final = "backup" + expensive_model: Final = "expensive" + monkeypatch.setattr(litellm, "model_alias_map", global_alias_map) + valid_token: Final = UserAPIKeyAuth( + token="test-key", + user_id="user-1", + team_id="team-1", + aliases=key_aliases, + team_member_model_max_budget={ + primary_model: {"max_budget": 5.0, "budget_duration": "1d"}, + expensive_model: {"max_budget": 5.0, "budget_duration": "1d"}, + }, + ) + limiter, redis_cache = await _real_model_budget_limiter( + member_spend={primary_model: 10.0, expensive_model: 10.0} + ) + request_data: Final[dict[str, object]] = {"model": primary_model} + request: Final = self._request(primary_model) + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=[primary_model], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{primary_model: [fallback_model]}]), + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == primary_model + assert request.scope["path_params"]["model"] == primary_model + assert [ + call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list + ] == _team_member_model_budget_spend_keys([primary_model, expensive_model]) + + @pytest.mark.asyncio + async def test_free_fallback_aliased_to_spent_paid_model_is_skipped(self) -> None: + primary_model: Final = "claude-sonnet-4-6" + fallback_model: Final = "free-backup" + expensive_model: Final = "expensive" + valid_token: Final = UserAPIKeyAuth( + token="test-key", + user_id="user-1", + team_id="team-1", + aliases={fallback_model: expensive_model}, + team_member_model_max_budget={ + primary_model: {"max_budget": 5.0, "budget_duration": "1d"}, + expensive_model: {"max_budget": 5.0, "budget_duration": "1d"}, + }, + ) + limiter, redis_cache = await _real_model_budget_limiter( + member_spend={primary_model: 10.0, expensive_model: 10.0} + ) + request_data: Final[dict[str, object]] = {"model": primary_model} + request: Final = self._request(primary_model) + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=[primary_model], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router( + [{primary_model: [fallback_model]}], + free_models=frozenset({fallback_model}), + ), + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == primary_model + assert request.scope["path_params"]["model"] == primary_model + assert [ + call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list + ] == _team_member_model_budget_spend_keys([primary_model, expensive_model]) + + @pytest.mark.asyncio + async def test_fallback_rewrites_router_alias_to_budget_checked_target(self) -> None: + valid_token: Final = UserAPIKeyAuth( + token="test-key", + user_id="user-1", + team_id="team-1", + router_settings={"model_group_alias": {"backup": "expensive"}}, + team_member_model_max_budget={ + "claude-sonnet-4-6": {"max_budget": 5.0, "budget_duration": "1d"}, + "backup": {"max_budget": 100.0, "budget_duration": "1d"}, + "expensive": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + limiter, redis_cache = await _real_model_budget_limiter(member_spend={"claude-sonnet-4-6": 10.0}) + request_data: Final[dict[str, object]] = {"model": "claude-sonnet-4-6"} + request: Final = self._request("claude-sonnet-4-6") + + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["claude-sonnet-4-6"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"claude-sonnet-4-6": ["backup"]}]), + ) + + assert request_data["model"] == "expensive" + assert request.scope["path_params"]["model"] == "expensive" + assert [ + call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list + ] == _team_member_model_budget_spend_keys(["claude-sonnet-4-6", "backup", "expensive"]) + + @pytest.mark.asyncio + async def test_zero_cost_fallback_skips_spent_member_model_budget(self) -> None: + primary_model: Final = "claude-sonnet-4-6" + fallback_model: Final = "free-backup" + valid_token: Final = UserAPIKeyAuth( + token="test-key", + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={ + primary_model: {"max_budget": 5.0, "budget_duration": "1d"}, + fallback_model: {"max_budget": 5.0, "budget_duration": "1d"}, + }, + ) + limiter, redis_cache = await _real_model_budget_limiter( + member_spend={primary_model: 10.0, fallback_model: 10.0} + ) + request_data: Final[dict[str, object]] = {"model": primary_model} + request: Final = self._request(primary_model) + + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=[primary_model], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router( + [{primary_model: [fallback_model]}], + free_models=frozenset({fallback_model}), + ), + ) + + assert request_data["model"] == fallback_model + assert request.scope["path_params"]["model"] == fallback_model + assert [ + call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list + ] == _team_member_model_budget_spend_keys([primary_model]) + + @pytest.mark.asyncio + async def test_fallback_settings_lookup_error_keeps_original_budget_error(self) -> None: + import litellm.proxy.proxy_server as proxy_server + + valid_token: Final = UserAPIKeyAuth( + token="test-key", + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={"claude-sonnet-4-6": {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, redis_cache = await _real_model_budget_limiter(member_spend={"claude-sonnet-4-6": 10.0}) + request_data: Final = {"model": "claude-sonnet-4-6"} + request: Final = self._request("claude-sonnet-4-6") + await proxy_server.user_api_key_cache.async_delete_cache(key="team_id:team-1") + proxy_server.prisma_client.db.litellm_teamtable.find_unique = AsyncMock( + side_effect=RuntimeError("settings unavailable") + ) + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["claude-sonnet-4-6"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router(None), + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == "claude-sonnet-4-6" + assert [ + call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list + ] == _team_member_model_budget_spend_keys(["claude-sonnet-4-6"]) + proxy_server.prisma_client.db.litellm_teamtable.find_unique.assert_awaited_once() + + @pytest.mark.asyncio + async def test_fallback_skips_candidate_after_budget_check_error(self) -> None: + valid_token: Final = UserAPIKeyAuth( + token="test-key", + user_id="user-1", + team_id="team-1", + router_settings={"model_group_alias": {}}, + team_member_model_max_budget={ + "claude-sonnet-4-6": {"max_budget": 5.0, "budget_duration": "1d"}, + "backup-unavailable": {"max_budget": 100.0, "budget_duration": "1d"}, + "backup-valid": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + limiter, redis_cache = await _real_model_budget_limiter( + member_spend={"claude-sonnet-4-6": 10.0}, + member_errors={"backup-unavailable": RuntimeError("budget counter unavailable")}, + ) + request_data: Final = {"model": "claude-sonnet-4-6"} + request: Final = self._request("claude-sonnet-4-6") + + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["claude-sonnet-4-6"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"claude-sonnet-4-6": ["backup-unavailable", "backup-valid"]}]), + ) + + assert request_data["model"] == "backup-valid" + assert request.scope["path_params"]["model"] == "backup-valid" + assert [ + call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list + ] == _team_member_model_budget_spend_keys(["claude-sonnet-4-6", "backup-unavailable", "backup-valid"]) + + @pytest.mark.asyncio + async def test_fallback_candidate_check_errors_keep_original_budget_error(self) -> None: + valid_token: Final = UserAPIKeyAuth( + token="test-key", + user_id="user-1", + team_id="team-1", + router_settings={"model_group_alias": {}}, + team_member_model_max_budget={ + "claude-sonnet-4-6": {"max_budget": 5.0, "budget_duration": "1d"}, + "backup-unavailable": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + limiter, _ = await _real_model_budget_limiter( + member_spend={"claude-sonnet-4-6": 10.0}, + member_errors={"backup-unavailable": RuntimeError("budget counter unavailable")}, + ) + request_data: Final = {"model": "claude-sonnet-4-6"} + request: Final = self._request("claude-sonnet-4-6") + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["claude-sonnet-4-6"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"claude-sonnet-4-6": ["backup-unavailable"]}]), + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == "claude-sonnet-4-6" + + @pytest.mark.asyncio + async def test_fallback_outside_project_models_raises_original_member_budget_error( + self, + ) -> None: + valid_token: Final = UserAPIKeyAuth( + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={ + "claude-sonnet-4-6": {"max_budget": 5.0, "budget_duration": "1d"}, + "claude-haiku-4-5": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + limiter, _ = await _real_model_budget_limiter(member_spend={"claude-sonnet-4-6": 10.0}) + request_data: Final = {"model": "claude-sonnet-4-6"} + request: Final = self._request("claude-sonnet-4-6") + project_object: Final = LiteLLM_ProjectTableCachedObj( + project_id="project-1", + models=["claude-sonnet-4-6"], + ) + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["claude-sonnet-4-6"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"claude-sonnet-4-6": ["claude-haiku-4-5"]}]), + project_object=project_object, + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == "claude-sonnet-4-6" + + @pytest.mark.asyncio + async def test_fallback_within_project_models_rewrites_request( + self, + ) -> None: + valid_token: Final = UserAPIKeyAuth( + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={ + "claude-sonnet-4-6": {"max_budget": 5.0, "budget_duration": "1d"}, + "claude-haiku-4-5": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + limiter, _ = await _real_model_budget_limiter(member_spend={"claude-sonnet-4-6": 10.0}) + request_data: Final = {"model": "claude-sonnet-4-6"} + request: Final = self._request("claude-sonnet-4-6") + project_object: Final = LiteLLM_ProjectTableCachedObj( + project_id="project-1", + models=["claude-sonnet-4-6", "claude-haiku-4-5"], + ) + + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["claude-sonnet-4-6"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"claude-sonnet-4-6": ["claude-haiku-4-5"]}]), + project_object=project_object, + ) + + assert request_data["model"] == "claude-haiku-4-5" + assert request.scope["path_params"]["model"] == "claude-haiku-4-5" + + @pytest.mark.asyncio + @pytest.mark.parametrize("grant_source", ["team", "key"]) + async def test_fallback_uses_team_and_key_access_group_grants( + self, grant_source: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import LiteLLM_AccessGroupTable + + valid_token: Final = UserAPIKeyAuth( + token="test-key", + user_id="user-1", + team_id="team-1", + team_models=["claude-sonnet-4-6"], + access_group_ids=["key-group"] if grant_source == "key" else [], + team_member_model_max_budget={ + "claude-sonnet-4-6": {"max_budget": 5.0, "budget_duration": "1d"}, + "claude-haiku-4-5": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + team_object: Final = LiteLLM_TeamTable( + team_id="team-1", + models=["claude-sonnet-4-6"], + access_group_ids=["team-group"] if grant_source == "team" else [], + ) + limiter, _ = await _real_model_budget_limiter(member_spend={"claude-sonnet-4-6": 10.0}) + request_data: Final = {"model": "claude-sonnet-4-6"} + request: Final = self._request("claude-sonnet-4-6") + prisma_client: Final = MagicMock() + access_group_id: Final = "team-group" if grant_source == "team" else "key-group" + group_row: Final = LiteLLM_AccessGroupTable( + access_group_id=access_group_id, + access_group_name=access_group_id, + access_model_names=["claude-haiku-4-5"], + assigned_team_ids=["team-1"], + ) + prisma_client.db.litellm_accessgrouptable.find_unique = AsyncMock(return_value=group_row) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["claude-sonnet-4-6"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"claude-sonnet-4-6": ["claude-haiku-4-5"]}]), + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=proxy_server.user_api_key_cache, + proxy_logging_obj=proxy_server.proxy_logging_obj, + ) + + assert request_data["model"] == "claude-haiku-4-5" + prisma_client.db.litellm_accessgrouptable.find_unique.assert_awaited_once_with( + where={"access_group_id": access_group_id} + ) + + @pytest.mark.asyncio + async def test_fallback_without_team_or_key_access_group_grants_keeps_budget_error(self) -> None: + valid_token: Final = UserAPIKeyAuth( + user_id="user-1", + team_id="team-1", + team_models=["claude-sonnet-4-6"], + team_member_model_max_budget={ + "claude-sonnet-4-6": {"max_budget": 5.0, "budget_duration": "1d"}, + "claude-haiku-4-5": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + team_object: Final = LiteLLM_TeamTable(team_id="team-1", models=["claude-sonnet-4-6"]) + limiter, _ = await _real_model_budget_limiter(member_spend={"claude-sonnet-4-6": 10.0}) + request_data: Final = {"model": "claude-sonnet-4-6"} + request: Final = self._request("claude-sonnet-4-6") + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["claude-sonnet-4-6"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"claude-sonnet-4-6": ["claude-haiku-4-5"]}]), + team_object=team_object, + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == "claude-sonnet-4-6" + + @pytest.mark.asyncio + async def test_exhausted_fallback_raises_original_member_budget_error(self): + budget: Final = { + "gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}, + "gpt-4o-mini": {"max_budget": 5.0, "budget_duration": "1d"}, + } + valid_token: Final = UserAPIKeyAuth( + token="test-key", + user_id="user-1", + team_id="team-1", + team_member_model_max_budget=budget, + ) + limiter, _ = await _real_model_budget_limiter(member_spend={"gpt-4o": 10.0, "gpt-4o-mini": 10.0}) + request_data: Final = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"gpt-4o": ["gpt-4o-mini"]}]), + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == "gpt-4o" + + @pytest.mark.asyncio + async def test_no_router_fallback_raises_original_member_budget_error(self): + valid_token: Final = UserAPIKeyAuth( + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={"gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, _ = await _real_model_budget_limiter(member_spend={"gpt-4o": 10.0}) + request_data: Final = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=None, + ) + + _assert_budget_exceeded_values(exc_info.value) + + @pytest.mark.asyncio + async def test_router_without_fallbacks_raises_original_member_budget_error(self): + budget: Final = {"gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}} + valid_token: Final = UserAPIKeyAuth( + user_id="user-1", + team_id="team-1", + team_member_model_max_budget=budget, + ) + limiter, _ = await _real_model_budget_limiter(member_spend={"gpt-4o": 10.0}) + request_data: Final = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router(None), + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == "gpt-4o" + + @pytest.mark.asyncio + async def test_fallback_exceeding_key_budget_is_rejected(self): + valid_token: Final = UserAPIKeyAuth( + token="test-key", + model_max_budget={"gpt-4o-mini": {"max_budget": 1.0, "budget_duration": "1d"}}, + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={"gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, redis_cache = await _real_model_budget_limiter( + member_spend={"gpt-4o": 10.0}, key_spend={"gpt-4o-mini": 2.0} + ) + request_data: Final = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"gpt-4o": ["gpt-4o-mini"]}]), + ) + + _assert_budget_exceeded_values(exc_info.value) + assert [ + call.kwargs["key"] + for call in redis_cache.async_get_cache.call_args_list + if call.kwargs["key"].startswith("virtual_key_spend:") + ] == _model_budget_spend_keys(Litellm_EntityType.KEY, "test-key", ["gpt-4o-mini"]) + + @pytest.mark.asyncio + async def test_fallback_outside_key_models_is_rejected(self): + valid_token: Final = UserAPIKeyAuth( + token="test-key", + models=["gpt-4o"], + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={"gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, _ = await _real_model_budget_limiter(member_spend={"gpt-4o": 10.0}) + request_data: Final = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + denied: Final = ProxyException( + message="model not allowed", + type=ProxyErrorTypes.budget_exceeded, + param="model", + code=status.HTTP_403_FORBIDDEN, + ) + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"gpt-4o": ["gpt-4o-mini"]}]), + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == "gpt-4o" + + @pytest.mark.asyncio + async def test_agent_key_does_not_reroute_to_fallback(self): + valid_token: Final = UserAPIKeyAuth( + token="test-key", + models=["gpt-4o", "gpt-4o-mini"], + user_id="user-1", + team_id="team-1", + agent_id="agent-1", + team_member_model_max_budget={ + "gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}, + "gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + limiter, _ = await _real_model_budget_limiter(member_spend={"gpt-4o": 10.0}) + request_data: Final = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"gpt-4o": ["gpt-4o-mini"]}]), + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == "gpt-4o" + assert getattr(request, "_json", None) is None + + @pytest.mark.asyncio + async def test_agent_key_can_reroute_on_key_model_budget_fallback(self): + valid_token: Final = UserAPIKeyAuth( + token="test-key", + models=["gpt-4o", "gpt-4o-mini"], + agent_id="agent-1", + budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]}, + model_max_budget={ + "gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}, + "gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + limiter, _ = await _real_model_budget_limiter(key_spend={"gpt-4o": 10.0}) + request_data: Final = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + + await _check_key_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + model_name="gpt-4o", + request_data=request_data, + request=request, + ) + + assert request_data["model"] == "gpt-4o-mini" + + @pytest.mark.asyncio + async def test_fallback_outside_member_allowed_models_is_rejected(self): + valid_token: Final = UserAPIKeyAuth( + token="test-key", + models=["gpt-4o", "gpt-4o-mini"], + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={"gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, _ = await _real_model_budget_limiter(member_spend={"gpt-4o": 10.0}) + request_data: Final = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + import litellm.proxy.proxy_server as proxy_server + + membership: Final = LiteLLM_TeamMembership( + user_id="user-1", + team_id="team-1", + budget_id="member-budget", + litellm_budget_table=LiteLLM_BudgetTable(budget_id="member-budget", allowed_models=["gpt-4o"]), + ) + await proxy_server.user_api_key_cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id="user-1", team_id="team-1"), + value=membership, + model_type=LiteLLM_TeamMembership, + ) + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"gpt-4o": ["gpt-4o-mini"]}]), + ) + + _assert_budget_exceeded_values(exc_info.value) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("targets", "expected_model"), + [ + ([{"model": "gpt-4o-mini"}], "gpt-4o-mini"), + ([{"model": "gpt-4o-mini", "temperature": 0.2}, {"model": "claude-haiku"}], "claude-haiku"), + ([{"model": "gpt-4o-mini", "temperature": 0.2}], None), + ], + ) + async def test_dict_fallback_targets_only_reroute_model_only_entries(self, targets, expected_model): + valid_token: Final = UserAPIKeyAuth( + token="test-key", + models=["gpt-4o", "gpt-4o-mini", "claude-haiku"], + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={ + "gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}, + "gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}, + "claude-haiku": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + limiter, _ = await _real_model_budget_limiter(member_spend={"gpt-4o": 10.0}) + request_data: Final = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + + if expected_model is not None: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"gpt-4o": targets}]), + ) + assert request_data["model"] == expected_model + else: + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"gpt-4o": targets}]), + ) + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == "gpt-4o" + + @pytest.mark.asyncio + async def test_fallback_skips_candidate_denied_to_end_user(self): + valid_token: Final = UserAPIKeyAuth( + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={ + "gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}, + "gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + limiter, _ = await _real_model_budget_limiter(member_spend={"gpt-4o": 10.0}) + request_data: Final = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=self._router([{"gpt-4o": ["gpt-4o-mini"]}]), + end_user_object=SimpleNamespace(models=["gpt-4o"]), + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == "gpt-4o" + + @pytest.mark.asyncio + async def test_fallback_skips_candidates_without_pricing(self, monkeypatch: pytest.MonkeyPatch): + valid_token: Final = UserAPIKeyAuth( + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={"gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, redis_cache = await _real_model_budget_limiter(member_spend={"gpt-4o": 10.0}) + request_data: Final = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + router: Final = litellm.Router( + model_list=[ + {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}}, + {"model_name": "unpriced-model", "litellm_params": {"model": "openai/unpriced-model"}}, + ], + fallbacks=[{"gpt-4o": ["unpriced-model"]}], + ) + monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True) + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=router, + route="/v1/chat/completions", + ) + + _assert_budget_exceeded_values(exc_info.value) + assert [ + call.kwargs["key"] for call in redis_cache.async_get_cache.call_args_list + ] == _team_member_model_budget_spend_keys(["gpt-4o"]) + + @pytest.mark.asyncio + async def test_fallback_checks_candidate_access_group_budget_and_stamps_candidate_groups(self): + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.user_api_key_cache import model_access_group_cache_key + from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget + + valid_token: Final = UserAPIKeyAuth( + token="test-key", + models=["gpt-4o", "gpt-4o-mini", "fallback-group"], + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={ + "gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}, + "gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + limiter, _ = await _real_model_budget_limiter(member_spend={"gpt-4o": 10.0}) + request_data: Final = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + await proxy_server.user_api_key_cache.async_set_cache( + key=model_access_group_cache_key("fallback-group"), + value=ModelAccessGroupBudget(access_group_name="fallback-group", spend=0.0, max_budget=10.0), + model_type=ModelAccessGroupBudget, + ) + router: Final = litellm.Router( + model_list=[ + {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}}, + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "gpt-4o-mini"}, + "model_info": {"access_groups": ["fallback-group"]}, + }, + ], + fallbacks=[{"gpt-4o": ["gpt-4o-mini"]}], + ) + + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=router, + ) + + assert request_data["model"] == "gpt-4o-mini" + assert request.scope["path_params"]["model"] == "gpt-4o-mini" + assert valid_token.matched_model_access_groups == ["fallback-group"] + + @pytest.mark.asyncio + async def test_access_group_budget_exhaustion_skips_fallback_candidate(self): + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.user_api_key_cache import model_access_group_cache_key + from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget + + valid_token: Final = UserAPIKeyAuth( + token="test-key", + models=["gpt-4o", "gpt-4o-mini", "fallback-group"], + user_id="user-1", + team_id="team-1", + team_member_model_max_budget={ + "gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}, + "gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + limiter, _ = await _real_model_budget_limiter(member_spend={"gpt-4o": 10.0}) + request_data: Final = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + await proxy_server.user_api_key_cache.async_set_cache( + key=model_access_group_cache_key("fallback-group"), + value=ModelAccessGroupBudget(access_group_name="fallback-group", spend=5.0, max_budget=5.0), + model_type=ModelAccessGroupBudget, + ) + router: Final = litellm.Router( + model_list=[ + {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}}, + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "gpt-4o-mini"}, + "model_info": {"access_groups": ["fallback-group"]}, + }, + ], + fallbacks=[{"gpt-4o": ["gpt-4o-mini"]}], + ) + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_member_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=router, + ) + + _assert_budget_exceeded_values(exc_info.value) + assert request_data["model"] == "gpt-4o" + + @pytest.mark.asyncio async def test_global_proxy_spend_reads_resettable_proxy_budget_row(): """Regression for the global proxy budget ignoring budget_duration @@ -10185,9 +12040,7 @@ async def test_managed_agent_cannot_bypass_grants_with_server_default( monkeypatch.setattr(proxy_server, name, value) data = {"messages": [{"role": "user", "content": "hi"}], **({"model": requested} if requested else {})} auth = UserAPIKeyAuth(agent_id="managed") - auth.managed_agent_context = ManagedAgentContext( - agent_id="managed", binding_revision="revision", mode="autonomous" - ) + auth.managed_agent_context = ManagedAgentContext(agent_id="managed", binding_revision="revision", mode="autonomous") if not grant_default: with pytest.raises(ProxyException) as denied: await _authorize_authenticated_request(auth, _alias_request(route, data), data, route, "persisted-key") @@ -10217,12 +12070,21 @@ async def test_managed_jwt_cannot_be_downgraded_into_virtual_key_mapping(monkeyp from litellm.types.proxy.agent_identity import AgentIdentityBinding binding: Final = AgentIdentityBinding( - agent_id="managed", provider="microsoft_entra", issuer="issuer", tenant_id="tenant", - client_id="client", service_principal_id="principal", revision="current", + agent_id="managed", + provider="microsoft_entra", + issuer="issuer", + tenant_id="tenant", + client_id="client", + service_principal_id="principal", + revision="current", ) agent: Final = AgentResponse( - agent_id="managed", agent_name="Managed", agent_card_params={}, - identity_managed=True, identity=binding, execution_mode="autonomous", + agent_id="managed", + agent_name="Managed", + agent_card_params={}, + identity_managed=True, + identity=binding, + execution_mode="autonomous", ) client: Final = MagicMock() client.writer_db.litellm_agentidentity.find_unique = AsyncMock(return_value=binding) @@ -10230,22 +12092,35 @@ async def test_managed_jwt_cannot_be_downgraded_into_virtual_key_mapping(monkeyp handler: Final = MagicMock() handler.is_jwt.return_value = True handler.litellm_jwtauth = LiteLLM_JWTAuth(virtual_key_claim_field="sub") - handler.auth_jwt = AsyncMock(return_value={ - "iss": "issuer", "tid": "tenant", "azp": "client", "oid": "principal", "sub": "mapped-key", - }) + handler.auth_jwt = AsyncMock( + return_value={ + "iss": "issuer", + "tid": "tenant", + "azp": "client", + "oid": "principal", + "sub": "mapped-key", + } + ) for name, value in { **_proxy_attrs_for_centralized_checks(), - "general_settings": {"enable_jwt_auth": True}, "premium_user": True, - "prisma_client": client, "jwt_handler": handler, "user_api_key_cache": UserApiKeyCache(), + "general_settings": {"enable_jwt_auth": True}, + "premium_user": True, + "prisma_client": client, + "jwt_handler": handler, + "user_api_key_cache": UserApiKeyCache(), "proxy_logging_obj": MagicMock(post_call_failure_hook=AsyncMock(return_value=None)), }.items(): monkeypatch.setattr(proxy_server, name, value) for _ in range(2): with pytest.raises(ProxyException) as failure: await user_api_key_auth_builder( - request=_alias_request("/v1/chat/completions", {}), api_key="Bearer verified.jwt.token", - azure_api_key_header="", anthropic_api_key_header=None, google_ai_studio_api_key_header=None, - azure_apim_header=None, request_data={}, + request=_alias_request("/v1/chat/completions", {}), + api_key="Bearer verified.jwt.token", + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data={}, ) assert failure.value.code == "403" assert "without virtual-key mapping" in failure.value.message @@ -10262,9 +12137,17 @@ async def test_virtual_key_cannot_enter_checks_as_an_identity_managed_actor(monk from litellm.types.proxy.agent_identity import AgentIdentityBinding target: Final = AgentResponse( - agent_id="bound", agent_name="Bound", agent_card_params={}, identity_managed=True, + agent_id="bound", + agent_name="Bound", + agent_card_params={}, + identity_managed=True, identity=AgentIdentityBinding( - agent_id="bound", provider="microsoft_entra", tenant_id="tenant", client_id="client", issuer="issuer", revision="current" + agent_id="bound", + provider="microsoft_entra", + tenant_id="tenant", + client_id="client", + issuer="issuer", + revision="current", ), ) client: Final = MagicMock() @@ -10272,7 +12155,9 @@ async def test_virtual_key_cannot_enter_checks_as_an_identity_managed_actor(monk monkeypatch.setattr(proxy_server, "prisma_client", client) checks: Final = AsyncMock() monkeypatch.setattr(auth_module, "run_centralized_common_checks", checks) - monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock(post_call_failure_hook=AsyncMock(return_value=None))) + monkeypatch.setattr( + proxy_server, "proxy_logging_obj", MagicMock(post_call_failure_hook=AsyncMock(return_value=None)) + ) data: Final = {"model": "allowed", "messages": [{"role": "user", "content": "hello"}]} request: Final = _alias_request("/v1/chat/completions", data) with pytest.raises(ProxyException): @@ -10300,16 +12185,24 @@ async def test_custom_auth_grants_reach_managed_targets_without_a_virtual_key_ro from litellm.types.proxy.agent_identity import AgentIdentityBinding target: Final = AgentResponse( - agent_id="target", agent_name="Target", agent_card_params={}, identity_managed=True, + agent_id="target", + agent_name="Target", + agent_card_params={}, + identity_managed=True, identity=AgentIdentityBinding( - agent_id="target", provider="microsoft_entra", tenant_id="tenant", client_id="client", - issuer="issuer", revision="current", + agent_id="target", + provider="microsoft_entra", + tenant_id="tenant", + client_id="client", + issuer="issuer", + revision="current", ), ) registry: Final = AgentRegistry() registry.register_agent(target) trusted: Final = UserAPIKeyAuth( - api_key=credential, object_permission={"object_permission_id": "custom", "agents": ["target"] if granted else ["other"]} + api_key=credential, + object_permission={"object_permission_id": "custom", "agents": ["target"] if granted else ["other"]}, ) custom: Final = AsyncMock(return_value=trusted) database: Final = MagicMock() @@ -10325,9 +12218,13 @@ async def test_custom_auth_grants_reach_managed_targets_without_a_virtual_key_ro monkeypatch.setattr(agent_registry, "global_agent_registry", registry) monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", False, raising=False) admitted: Final = await user_api_key_auth_builder( - request=_alias_request("/a2a/target/message/send", {}), api_key=f"Bearer {credential}", - azure_api_key_header="", anthropic_api_key_header=None, google_ai_studio_api_key_header=None, - azure_apim_header=None, request_data={}, + request=_alias_request("/a2a/target/message/send", {}), + api_key=f"Bearer {credential}", + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data={}, ) assert await AgentRequestHandler.is_agent_allowed("target", admitted) is granted custom.assert_awaited_once() @@ -10347,9 +12244,13 @@ async def test_enterprise_custom_auth_key_return_stays_a_proxy_validated_key(mon module: Final = importlib.import_module("litellm.proxy.auth.user_api_key_auth") monkeypatch.setattr(module, "enterprise_custom_auth", custom) admitted: Final = await user_api_key_auth_builder( - request=_alias_request("/v1/chat/completions", {}), api_key="Bearer external-credential", - azure_api_key_header="", anthropic_api_key_header=None, google_ai_studio_api_key_header=None, - azure_apim_header=None, request_data={}, + request=_alias_request("/v1/chat/completions", {}), + api_key="Bearer external-credential", + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data={}, ) assert admitted.authenticated_by_custom_auth is False assert admitted.via_virtual_key is True diff --git a/tests/unit/proxy/common_utils/test_upsert_budget_membership.py b/tests/unit/proxy/common_utils/test_upsert_budget_membership.py index 74d1748d4de..1f7154f0a74 100644 --- a/tests/unit/proxy/common_utils/test_upsert_budget_membership.py +++ b/tests/unit/proxy/common_utils/test_upsert_budget_membership.py @@ -1,6 +1,8 @@ # tests/litellm/proxy/common_utils/test_upsert_budget_membership.py +import json import types from datetime import datetime, timedelta, timezone +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -230,7 +232,11 @@ async def test_create_from_temp_budget_pair_only(mock_tx, fake_user): ) mock_tx.litellm_budgettable.create.assert_awaited_once() - assert stored_budget_row(mock_tx) == {"temp_budget_increase": 5.0, "temp_budget_expiry": expiry} + assert stored_budget_row(mock_tx) == { + "model_max_budget": "{}", + "temp_budget_increase": 5.0, + "temp_budget_expiry": expiry, + } mock_tx.litellm_teammembership.upsert.assert_awaited_once() mock_tx.litellm_teammembership.update.assert_not_called() @@ -252,7 +258,11 @@ async def test_create_from_temp_pair_never_snapshots_team_default(mock_tx, fake_ ) mock_tx.litellm_budgettable.find_unique.assert_not_awaited() - assert stored_budget_row(mock_tx) == {"temp_budget_increase": 1.0, "temp_budget_expiry": expiry} + assert stored_budget_row(mock_tx) == { + "model_max_budget": "{}", + "temp_budget_increase": 1.0, + "temp_budget_expiry": expiry, + } mock_tx.litellm_teammembership.upsert.assert_awaited_once() @@ -274,7 +284,11 @@ async def test_temp_pair_on_shared_default_member_creates_bare_row(mock_tx, fake mock_tx.litellm_budgettable.find_unique.assert_not_awaited() mock_tx.litellm_budgettable.update.assert_not_called() - assert stored_budget_row(mock_tx) == {"temp_budget_increase": 1.0, "temp_budget_expiry": expiry} + assert stored_budget_row(mock_tx) == { + "model_max_budget": "{}", + "temp_budget_increase": 1.0, + "temp_budget_expiry": expiry, + } mock_tx.litellm_teammembership.upsert.assert_awaited_once() @@ -296,6 +310,58 @@ async def test_clearing_temp_pair_on_shared_default_member_is_noop(mock_tx, fake mock_tx.litellm_teammembership.upsert.assert_not_called() +@pytest.mark.asyncio +async def test_clearing_model_budget_to_none_on_shared_default_is_noop(mock_tx, fake_user): + await upsert_budget_and_membership( + mock_tx, + team_id="team-default", + user_id="user-on-default", + existing_budget_id="team-default-budget-1", + user_api_key_dict=fake_user, + budget_patch={"model_max_budget": None}, + team_default_budget_id="team-default-budget-1", + ) + + mock_tx.litellm_budgettable.find_unique.assert_not_awaited() + mock_tx.litellm_budgettable.create.assert_not_called() + mock_tx.litellm_budgettable.update.assert_not_called() + mock_tx.litellm_teammembership.update.assert_not_called() + mock_tx.litellm_teammembership.upsert.assert_not_called() + + +@pytest.mark.asyncio +async def test_clearing_model_budget_to_none_with_tpm_clones_shared_default(mock_tx, fake_user): + shared_default_id: Final = "team-default-budget-1" + mock_tx.litellm_budgettable.find_unique = AsyncMock( + return_value=budget_row( + budget_id=shared_default_id, + max_budget=100.0, + model_max_budget={"claude-sonnet-4-6": {"max_budget": 5.0}}, + ) + ) + + await upsert_budget_and_membership( + mock_tx, + team_id="team-default", + user_id="user-on-default", + existing_budget_id=shared_default_id, + user_api_key_dict=fake_user, + budget_patch={"model_max_budget": None, "tpm_limit": 5}, + team_default_budget_id=shared_default_id, + ) + + create_data: Final = mock_tx.litellm_budgettable.create.await_args.kwargs["data"] + assert create_data == { + "created_by": fake_user.user_id, + "updated_by": fake_user.user_id, + "model_max_budget": "{}", + "max_budget": 100.0, + "tpm_limit": 5, + } + mock_tx.litellm_budgettable.update.assert_not_called() + mock_tx.litellm_teammembership.upsert.assert_awaited_once() + + @pytest.mark.asyncio async def test_temp_pair_with_permanent_field_still_clones_shared_default(mock_tx, fake_user): expiry = datetime(2100, 1, 1, tzinfo=timezone.utc) @@ -337,7 +403,7 @@ async def test_create_from_plain_patch_does_not_snapshot_team_default(mock_tx, f ) mock_tx.litellm_budgettable.find_unique.assert_not_awaited() - assert stored_budget_row(mock_tx) == {"tpm_limit": 500} + assert stored_budget_row(mock_tx) == {"model_max_budget": "{}", "tpm_limit": 500} mock_tx.litellm_teammembership.upsert.assert_awaited_once() @@ -380,6 +446,7 @@ async def test_clone_on_write_from_shared_default(mock_tx, fake_user): assert create_data == { "created_by": fake_user.user_id, "updated_by": fake_user.user_id, + "model_max_budget": "{}", "max_budget": 50.0, # caller wins "tpm_limit": 500, # cloned from default "budget_duration": "1d", # cloned from default @@ -401,6 +468,69 @@ async def test_clone_on_write_from_shared_default(mock_tx, fake_user): ) +@pytest.mark.asyncio +async def test_clone_on_write_preserves_model_caps_from_shared_nondefault_row(mock_tx, fake_user): + shared_budget_id: Final = "shared-member-budget" + model_caps: Final = {"gpt-4o": {"max_budget": 4.0, "budget_duration": "1d"}} + mock_tx.litellm_budgettable.find_unique = AsyncMock( + return_value=budget_row( + budget_id=shared_budget_id, + max_budget=100.0, + model_max_budget=model_caps, + ) + ) + + await upsert_budget_and_membership( + mock_tx, + team_id="team-shared", + user_id="user-shared", + existing_budget_id=shared_budget_id, + user_api_key_dict=fake_user, + budget_patch={"tpm_limit": 1000}, + team_default_budget_id="team-default-budget", + shared_budget_ids=frozenset({shared_budget_id}), + ) + + create_data: Final = mock_tx.litellm_budgettable.create.await_args.kwargs["data"] + persisted_model_caps: Final = create_data["model_max_budget"] + assert ( + json.loads(persisted_model_caps) if isinstance(persisted_model_caps, str) else persisted_model_caps + ) == model_caps + + +@pytest.mark.asyncio +async def test_clearing_model_budget_to_none_clones_shared_nondefault_row(mock_tx, fake_user): + shared_budget_id: Final = "shared-member-budget" + mock_tx.litellm_budgettable.find_unique = AsyncMock( + return_value=budget_row( + budget_id=shared_budget_id, + max_budget=100.0, + model_max_budget={"gpt-4o": {"max_budget": 4.0}}, + ) + ) + + await upsert_budget_and_membership( + mock_tx, + team_id="team-shared", + user_id="user-shared", + existing_budget_id=shared_budget_id, + user_api_key_dict=fake_user, + budget_patch={"model_max_budget": None}, + team_default_budget_id="team-default-budget", + shared_budget_ids=frozenset({shared_budget_id}), + ) + + create_data: Final = mock_tx.litellm_budgettable.create.await_args.kwargs["data"] + assert create_data == { + "created_by": fake_user.user_id, + "updated_by": fake_user.user_id, + "model_max_budget": "{}", + "max_budget": 100.0, + } + mock_tx.litellm_budgettable.update.assert_not_called() + mock_tx.litellm_teammembership.upsert.assert_awaited_once() + + # TEST: forking the shared default while clearing its duration must drop the # duration (and not carry a reset time) on the new private budget. @pytest.mark.asyncio @@ -431,6 +561,7 @@ async def test_clone_on_write_clears_duration(mock_tx, fake_user): assert create_data == { "created_by": fake_user.user_id, "updated_by": fake_user.user_id, + "model_max_budget": "{}", "max_budget": 200.0, "tpm_limit": 500, "budget_duration": None, @@ -460,3 +591,31 @@ async def test_private_budget_updates_in_place(mock_tx, fake_user): ) mock_tx.litellm_budgettable.create.assert_not_called() mock_tx.litellm_teammembership.upsert.assert_not_called() + + +@pytest.mark.asyncio +async def test_clearing_model_budget_to_none_updates_private_row(mock_tx, fake_user): + mock_tx.litellm_budgettable.find_unique = AsyncMock( + return_value=budget_row( + max_budget=10.0, + model_max_budget={"gpt-4o": {"max_budget": 4.0}}, + ) + ) + + await upsert_budget_and_membership( + mock_tx, + team_id="team-mixed", + user_id="user-private", + existing_budget_id="private-budget-xyz", + user_api_key_dict=fake_user, + budget_patch={"model_max_budget": None}, + team_default_budget_id="team-default-budget-1", + ) + + mock_tx.litellm_budgettable.update.assert_awaited_once_with( + where={"budget_id": "private-budget-xyz"}, + data={"updated_by": fake_user.user_id, "model_max_budget": "{}"}, + ) + mock_tx.litellm_budgettable.create.assert_not_called() + mock_tx.litellm_teammembership.update.assert_not_called() + mock_tx.litellm_teammembership.upsert.assert_not_called() diff --git a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py index 8de34e6391e..2678f4205d5 100644 --- a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py +++ b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py @@ -6,15 +6,20 @@ from typing import Final import pytest +import litellm from litellm.caching.caching import DualCache +from litellm.proxy._types import Litellm_EntityType from litellm.proxy.hooks.model_max_budget_limiter import ( PROXY_VirtualKeyModelMaxBudgetLimiter, + model_budget_spend_cache_key, + team_member_budget_entity_id, ) from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.utils import LiteLLMBatch, Usage KEY_HASH: Final = "key-hash-batch" USER_ID: Final = "user-batch" +TEAM_ID: Final = "team-batch" MODEL_GROUP: Final = "batch-qa-primary" BATCH_COST: Final = 2.925e-05 CHAT_COST: Final = 0.001 @@ -35,7 +40,13 @@ def _batch(batch_id: str, status: str) -> LiteLLMBatch: ) -def _event(call_type: str, response_cost: float) -> dict[str, object]: +def _event( + call_type: str, + response_cost: float, + team_member_model_max_budget: Mapping[str, object] | None = None, + user_id: str = USER_ID, + team_id: str = TEAM_ID, +) -> dict[str, object]: return { "call_type": call_type, "standard_logging_object": { @@ -43,12 +54,21 @@ def _event(call_type: str, response_cost: float) -> dict[str, object]: "response_cost": response_cost, "model": "openai/gpt-5.4-mini", "model_group": MODEL_GROUP, - "metadata": {"user_api_key_hash": KEY_HASH, "user_api_key_user_id": USER_ID}, + "metadata": { + "user_api_key_hash": KEY_HASH, + "user_api_key_user_id": user_id, + "user_api_key_team_id": team_id, + }, }, "litellm_params": { "metadata": { "user_api_key_model_max_budget": {MODEL_GROUP: {"budget_limit": 0.0001, "time_period": "1d"}}, "user_api_key_user_model_max_budget": {MODEL_GROUP: {"budget_limit": 0.0001, "time_period": "1d"}}, + **( + {"user_api_key_team_member_model_max_budget": team_member_model_max_budget} + if team_member_model_max_budget is not None + else {} + ), } }, } @@ -60,9 +80,23 @@ async def _poll(limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, batch: LiteLLMBa ) -async def _chat(limiter: PROXY_VirtualKeyModelMaxBudgetLimiter) -> None: +async def _chat( + limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, + team_member_model_max_budget: Mapping[str, object] | None = None, + user_id: str = USER_ID, + team_id: str = TEAM_ID, +) -> None: await limiter.async_log_success_event( - _event("acompletion", CHAT_COST), response_obj=None, start_time=None, end_time=None + _event( + "acompletion", + CHAT_COST, + team_member_model_max_budget=team_member_model_max_budget, + user_id=user_id, + team_id=team_id, + ), + response_obj=None, + start_time=None, + end_time=None, ) @@ -74,6 +108,87 @@ def _local_spend(limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, spend_key: str) return limiter.dual_cache.in_memory_cache.get_cache(key=spend_key) or 0.0 +@pytest.mark.asyncio +async def test_team_member_budget_is_charged_and_enforced_after_success(): + budget: Final = {MODEL_GROUP: {"max_budget": 0.0005, "budget_duration": "1d"}} + limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache()) + + await _chat(limiter, team_member_model_max_budget=budget) + + entity_id: Final = team_member_budget_entity_id(user_id=USER_ID, team_id=TEAM_ID) + spend_key: Final = model_budget_spend_cache_key( + entity_type=Litellm_EntityType.TEAM_MEMBER, + entity_id=entity_id, + budget_model=MODEL_GROUP, + budget_duration="1d", + ) + assert await _spend(limiter, spend_key) == CHAT_COST + + with pytest.raises(litellm.BudgetExceededError, match="LiteLLM Team Member"): + await limiter.is_team_member_within_model_budget( + user_id=USER_ID, + team_id=TEAM_ID, + team_member_model_max_budget=budget, + model=MODEL_GROUP, + ) + + +def test_team_member_budget_entity_id_is_unambiguous(): + first_entity_id: Final = team_member_budget_entity_id(user_id="a:b", team_id="c") + second_entity_id: Final = team_member_budget_entity_id(user_id="a", team_id="b:c") + + assert first_entity_id == "3:a:b:c" + assert second_entity_id == "1:a:b:c" + + first_spend_key: Final = model_budget_spend_cache_key( + entity_type=Litellm_EntityType.TEAM_MEMBER, + entity_id=first_entity_id, + budget_model=MODEL_GROUP, + budget_duration="1d", + ) + second_spend_key: Final = model_budget_spend_cache_key( + entity_type=Litellm_EntityType.TEAM_MEMBER, + entity_id=second_entity_id, + budget_model=MODEL_GROUP, + budget_duration="1d", + ) + assert first_spend_key != second_spend_key + + +@pytest.mark.asyncio +async def test_team_member_budget_spend_isolated_by_user_and_team(): + budget: Final = {MODEL_GROUP: {"max_budget": CHAT_COST * 2, "budget_duration": "1d"}} + limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache()) + + await _chat(limiter, team_member_model_max_budget=budget) + + assert await limiter.is_team_member_within_model_budget( + user_id="different-user", + team_id=TEAM_ID, + team_member_model_max_budget=budget, + model=MODEL_GROUP, + ) + assert await limiter.is_team_member_within_model_budget( + user_id=USER_ID, + team_id="different-team", + team_member_model_max_budget=budget, + model=MODEL_GROUP, + ) + + +@pytest.mark.asyncio +async def test_zero_team_member_model_budget_rejects_first_request(): + limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache()) + + with pytest.raises(litellm.BudgetExceededError, match="LiteLLM Team Member"): + await limiter.is_team_member_within_model_budget( + user_id=USER_ID, + team_id=TEAM_ID, + team_member_model_max_budget={MODEL_GROUP: {"max_budget": 0, "budget_duration": "1d"}}, + model=MODEL_GROUP, + ) + + class _Clock: def __init__(self) -> None: self.seconds = 0.0 diff --git a/tests/unit/proxy/management_endpoints/management_v1/test_teams.py b/tests/unit/proxy/management_endpoints/management_v1/test_teams.py index 3f60384cfa7..5d1c658b143 100644 --- a/tests/unit/proxy/management_endpoints/management_v1/test_teams.py +++ b/tests/unit/proxy/management_endpoints/management_v1/test_teams.py @@ -85,6 +85,10 @@ def _matches(row: Mapping[str, object], where: Mapping[str, object]) -> bool: return all((wanted := _wanted(where, field)) is not None and row.get(field) in wanted for field in where) +def _as_stored(data: Mapping[str, object]) -> dict[str, object]: + return {k: json.loads(v) if k == "model_max_budget" and isinstance(v, str) else v for k, v in data.items()} + + class _BudgetTable: def __init__(self, budgets: Sequence[_BudgetRow]) -> None: self.rows: dict[str, _BudgetRow] = {b.budget_id: b for b in budgets} @@ -94,13 +98,13 @@ class _BudgetTable: async def update(self, where: Mapping[str, str], data: Mapping[str, object]) -> _BudgetRow: row: Final = self.rows[where["budget_id"]] - updated: Final = row.model_copy(update=dict(data)) + updated: Final = row.model_copy(update=_as_stored(data)) self.rows[row.budget_id] = updated return updated async def create(self, data: Mapping[str, object], include: Mapping[str, bool] | None = None) -> _BudgetRow: budget_id: Final = f"new-budget-{len(self.rows) + 1}" - row: Final = _BudgetRow.model_validate({**data, "budget_id": budget_id}) + row: Final = _BudgetRow.model_validate({**_as_stored(data), "budget_id": budget_id}) self.rows[budget_id] = row return row diff --git a/tests/unit/proxy/management_endpoints/test_budget_endpoints.py b/tests/unit/proxy/management_endpoints/test_budget_endpoints.py index 2f3be61d00f..24129488f21 100644 --- a/tests/unit/proxy/management_endpoints/test_budget_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_budget_endpoints.py @@ -11,6 +11,8 @@ from fastapi.testclient import TestClient import litellm.proxy.proxy_server as ps from litellm.proxy.proxy_server import app from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles, CommonProxyErrors +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache @@ -102,6 +104,25 @@ async def test_update_budget_rejects_a_duration_that_never_advances(client_and_m mock_table.update.assert_not_awaited() +@pytest.mark.asyncio +async def test_update_budget_evicts_local_cache_when_redis_delete_fails( + client_and_mocks, monkeypatch: pytest.MonkeyPatch +) -> None: + client, _, _ = client_and_mocks + cache_key = "team_member_default_budget:budget_456" + cache = UserApiKeyCache(in_memory_cache=InMemoryCache()) + cache.set_cache(cache_key, {"budget_id": "budget_456"}, local_only=True) + redis_cache = MagicMock() + redis_cache.async_delete_cache = AsyncMock(side_effect=RuntimeError("redis unavailable")) + cache.redis_cache = redis_cache + monkeypatch.setattr(ps, "user_api_key_cache", cache) + + response = client.post("/budget/update", json={"budget_id": "budget_456", "max_budget": 10.0}) + + assert response.status_code == 200, response.text + assert cache.get_cache(cache_key, local_only=True) is None + + @pytest.mark.asyncio async def test_new_budget_db_not_connected(client_and_mocks, monkeypatch): client, mock_prisma, mock_table = client_and_mocks @@ -136,6 +157,26 @@ async def test_update_budget_success(client_and_mocks, monkeypatch): assert body["updated_by"] == "test_user" +@pytest.mark.asyncio +async def test_update_budget_evicts_team_member_default_budget_cache(client_and_mocks, monkeypatch): + client, _, _ = client_and_mocks + cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache()) + monkeypatch.setattr(ps, "user_api_key_cache", cache) + cache_key: Final = "team_member_default_budget:budget_456" + await cache.async_set_cache(key=cache_key, value={"budget_id": "budget_456"}) + assert await cache.async_get_cache(key=cache_key) == {"budget_id": "budget_456"} + + try: + response: Final = client.post( + "/budget/update", + json={"budget_id": "budget_456", "max_budget": 10.0}, + ) + assert response.status_code == 200, response.text + assert await cache.async_get_cache(key=cache_key) is None + finally: + await cache.async_delete_cache(key=cache_key) + + @pytest.mark.asyncio async def test_new_and_update_budget_persist_tpd_limit(client_and_mocks): client, _, mock_table = client_and_mocks diff --git a/tests/unit/proxy/management_endpoints/test_common_utils.py b/tests/unit/proxy/management_endpoints/test_common_utils.py index d783f4c64f6..21d017addca 100644 --- a/tests/unit/proxy/management_endpoints/test_common_utils.py +++ b/tests/unit/proxy/management_endpoints/test_common_utils.py @@ -18,6 +18,7 @@ from pydantic import BaseModel from litellm import Router from litellm.proxy._types import ( + LiteLLM_BudgetTable, LiteLLM_OrganizationMembershipTable, LiteLLM_TeamTable, LiteLLM_UserTable, @@ -35,25 +36,132 @@ from litellm.proxy.management_endpoints.common_utils import ( update_metadata_fields, user_api_key_has_admin_view, user_has_admin_privileges, + upsert_budget_and_membership, ) from litellm.types.utils import BudgetConfig +@pytest.mark.asyncio +@pytest.mark.parametrize( + "budget_patch", + [ + {"tpm_limit": 200}, + {"model_max_budget": {}, "tpm_limit": 5}, + ], +) +async def test_shared_default_clone_keeps_inheriting_updated_team_model_caps( + budget_patch: dict[str, object], +) -> None: + from litellm.proxy.auth.user_api_key_auth import _load_team_member_default_model_budget + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, team_membership_reservation_cache_key + from litellm.proxy._types import LiteLLM_TeamMembership + + source_caps = {"claude-sonnet-4-6": {"max_budget": 3.0, "budget_duration": "1d"}} + default_caps = {"claude-sonnet-4-6": {"max_budget": 1.0, "budget_duration": "1d"}} + budget_table = SimpleNamespace( + find_unique=AsyncMock( + return_value=SimpleNamespace( + model_dump=MagicMock( + return_value={ + "budget_id": "default-budget", + "model_max_budget": source_caps, + "tpm_limit": 100, + } + ) + ) + ), + create=AsyncMock(return_value=SimpleNamespace(budget_id="private-budget")), + ) + membership_table = SimpleNamespace(upsert=AsyncMock()) + tx = SimpleNamespace(litellm_budgettable=budget_table, litellm_teammembership=membership_table) + token = UserAPIKeyAuth(user_id="member-1", team_id="team-1") + + await upsert_budget_and_membership( + tx, + team_id="team-1", + user_id="member-1", + existing_budget_id="default-budget", + user_api_key_dict=token, + budget_patch=budget_patch, + team_default_budget_id="default-budget", + ) + + budget_table.create.assert_awaited_once() + membership_table.upsert.assert_awaited_once() + assert budget_table.create.await_args.kwargs["data"]["tpm_limit"] == budget_patch["tpm_limit"] + model_max_budget: Final = budget_table.create.await_args.kwargs["data"]["model_max_budget"] + assert isinstance(model_max_budget, str) + assert model_max_budget == "{}" + membership = LiteLLM_TeamMembership( + user_id="member-1", + team_id="team-1", + budget_id="private-budget", + litellm_budget_table=LiteLLM_BudgetTable(budget_id="private-budget", model_max_budget={}), + ) + user_api_key_cache: Final = UserApiKeyCache() + await user_api_key_cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id="member-1", team_id="team-1"), + value=membership, + model_type=LiteLLM_TeamMembership, + ) + await user_api_key_cache.async_set_cache( + key="team_member_default_budget:default-budget", + value=LiteLLM_BudgetTable(budget_id="default-budget", model_max_budget=default_caps), + model_type=LiteLLM_BudgetTable, + ) + team_object = SimpleNamespace( + team_id="team-1", + metadata={"team_member_budget_id": "default-budget"}, + ) + + await _load_team_member_default_model_budget( + user_api_key_auth_obj=token, + team_object=team_object, + prisma_client=MagicMock(), + user_api_key_cache=user_api_key_cache, + skip_budget_checks=False, + proxy_logging_obj=None, + ) + + assert token.team_member_model_max_budget == default_caps + + +@pytest.mark.asyncio +async def test_empty_model_budget_patch_on_team_default_keeps_membership_attached() -> None: + tx: Final = MagicMock() + tx.litellm_budgettable.find_unique = AsyncMock( + return_value=SimpleNamespace( + model_dump=MagicMock( + return_value={ + "budget_id": "default-budget", + "max_budget": 10.0, + "model_max_budget": {"claude-sonnet-4-6": {"max_budget": 1.0}}, + } + ) + ) + ) + tx.litellm_budgettable.create = AsyncMock() + tx.litellm_budgettable.update = AsyncMock() + tx.litellm_teammembership.upsert = AsyncMock() + tx.litellm_teammembership.update = AsyncMock() + + await upsert_budget_and_membership( + tx, + team_id="team-1", + user_id="member-1", + existing_budget_id="default-budget", + user_api_key_dict=UserAPIKeyAuth(user_id="admin"), + budget_patch={"model_max_budget": {}}, + team_default_budget_id="default-budget", + ) + + tx.litellm_budgettable.create.assert_not_awaited() + tx.litellm_budgettable.update.assert_not_awaited() + tx.litellm_teammembership.upsert.assert_not_awaited() + tx.litellm_teammembership.update.assert_not_awaited() + + class TestUpdateMetadataFieldsEmptyCollections: - """ - Regression tests for issue #20304. - - The UI sends empty arrays (`[]`) for enterprise-only fields like - guardrails, policies, and logging even when the user hasn't configured - these features. The backend must not treat empty collections as an - intent to use the feature, and therefore must not trigger the premium - license check. - - However, empty collections must still be written into metadata so that - users can intentionally clear a previously-set field (e.g. removing all - guardrails by sending `guardrails: []`). - """ - @patch("litellm.proxy.management_endpoints.common_utils.premium_user_check") def test_empty_list_does_not_trigger_premium_check(self, mock_premium_check): """Empty lists for premium fields must not trigger the premium check.""" diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index b81c1c9f3d5..e921e4df5c1 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -2591,6 +2591,8 @@ async def test_update_team_team_member_budget_not_passed_to_db( team_member_rpm_limit=None, team_member_tpm_limit=None, team_member_budget_duration=None, + team_member_model_max_budget=None, + user_api_key_cache=None, explicitly_set_fields=frozenset(), ): # Remove team_member_budget from updated_kv as the real function does @@ -3159,6 +3161,8 @@ async def test_update_team_with_team_member_budget_duration( team_member_rpm_limit=None, team_member_tpm_limit=None, team_member_budget_duration=None, + team_member_model_max_budget=None, + user_api_key_cache=None, explicitly_set_fields=frozenset(), ): result_kv = updated_kv.copy() @@ -10891,6 +10895,7 @@ async def test_create_team_member_budget_table_with_duration(): team_alias="test-team", team_member_budget=20.0, team_member_budget_duration="30d", + team_member_model_max_budget={"gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}}, ) with patch( @@ -10904,15 +10909,181 @@ async def test_create_team_member_budget_table_with_duration(): user_api_key_dict=mock_admin, team_member_budget=20.0, team_member_budget_duration="30d", + team_member_model_max_budget=data.team_member_model_max_budget, ) mock_new_budget.assert_awaited_once() budget_request = mock_new_budget.call_args.kwargs["budget_obj"] assert budget_request.budget_duration == "30d" assert budget_request.max_budget == 20.0 + assert budget_request.model_max_budget == data.team_member_model_max_budget assert result["metadata"]["team_member_budget_id"] == "budget-abc" +@pytest.mark.asyncio +async def test_upsert_team_member_model_max_budget_updates_default_budget_and_evicts_cache(): + from litellm.proxy.management_endpoints.team_endpoints import TeamMemberBudgetHandler + + model_budget: Final = {"gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}} + team_table: Final = LiteLLM_TeamTable( + team_id="test-team", + metadata={"team_member_budget_id": "member-budget-1"}, + members_with_roles=[], + ) + cache: Final = MagicMock() + cache.async_delete_cache = AsyncMock() + budget_cache: Final = MagicMock() + budget_cache.async_delete_cache = AsyncMock() + prisma: Final = MagicMock() + prisma.db.litellm_budgettable.update = AsyncMock(return_value=MagicMock(budget_id="member-budget-1")) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), + patch("litellm.proxy.proxy_server.user_api_key_cache", budget_cache), + patch("litellm.proxy.proxy_server.premium_user", True), + ): + result: Final = await TeamMemberBudgetHandler.upsert_team_member_budget_table( + team_table=team_table, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"), + updated_kv={"team_id": "test-team", "team_member_model_max_budget": model_budget}, + team_member_model_max_budget=model_budget, + user_api_key_cache=cache, + explicitly_set_fields={"team_member_model_max_budget"}, + ) + + budget_update: Final = prisma.db.litellm_budgettable.update.call_args.kwargs + assert budget_update["where"]["budget_id"] == "member-budget-1" + assert budget_update["data"]["budget_id"] == "member-budget-1" + assert json.loads(budget_update["data"]["model_max_budget"]) == model_budget + assert result["metadata"]["team_member_budget_id"] == "member-budget-1" + assert "team_member_model_max_budget" not in result + cache.async_delete_cache.assert_awaited_once_with(key="team_member_default_budget:member-budget-1") + budget_cache.async_delete_cache.assert_awaited_once_with(key="team_member_default_budget:member-budget-1") + + +@pytest.mark.asyncio +async def test_update_team_clears_explicitly_null_member_model_budget_and_evicts_cache( + disable_audit_logging_for_mocked_team, +): + from fastapi import Request + + from litellm.proxy._types import UpdateTeamRequest + from litellm.proxy.management_endpoints.team_endpoints import update_team + + existing: Final = _existing_team_with_model_caps({}) + existing.metadata = {"team_member_budget_id": "member-budget-1"} + existing.members_with_roles = [] + existing.model_dump.return_value["metadata"] = existing.metadata + existing.model_dump.return_value["members_with_roles"] = [] + cache: Final = MagicMock() + cache.async_get_cache = AsyncMock(return_value=None) + cache.async_set_cache = AsyncMock() + cache.async_delete_cache = AsyncMock() + updated: Final = _existing_team_with_model_caps({}) + updated.metadata = {"team_member_budget_id": "member-budget-1"} + updated.model_dump.return_value.update({"metadata": updated.metadata, "members_with_roles": []}) + updated.litellm_model_table = None + update_budget_response: Final = MagicMock(budget_id="member-budget-1") + prisma: Final = MagicMock() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), + patch("litellm.proxy.proxy_server.user_api_key_cache", cache), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + patch("litellm.proxy.proxy_server.premium_user", True), + ): + prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=existing) + prisma.jsonify_team_object = lambda db_data: db_data + prisma.db.litellm_teamtable.update = AsyncMock(return_value=updated) + prisma.db.litellm_budgettable.update = AsyncMock(return_value=update_budget_response) + + request_data: Final = UpdateTeamRequest(team_id="standalone-team-123", team_member_model_max_budget=None) + await update_team( + data=request_data, + http_request=MagicMock(spec=Request), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"), + ) + + budget_update: Final = prisma.db.litellm_budgettable.update.call_args.kwargs + assert budget_update["where"]["budget_id"] == "member-budget-1" + assert budget_update["data"]["budget_id"] == "member-budget-1" + assert budget_update["data"]["model_max_budget"] == "{}" + assert "team_member_model_max_budget" in request_data.model_fields_set + assert "model_max_budget" in budget_update["data"] + assert cache.async_delete_cache.await_args_list == [ + call(key="team_member_default_budget:member-budget-1"), + call(key="team_member_default_budget:member-budget-1"), + ] + + +@pytest.mark.asyncio +async def test_update_team_persists_member_model_budget_to_default_budget_row( + disable_audit_logging_for_mocked_team, +): + from fastapi import Request + + from litellm.proxy._types import UpdateTeamRequest + from litellm.proxy.management_endpoints.team_endpoints import update_team + + member_caps: Final = {"gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}} + existing: Final = _existing_team_with_model_caps({}) + existing.metadata = {"team_member_budget_id": "member-budget-1"} + existing.members_with_roles = [] + existing.model_dump.return_value["metadata"] = existing.metadata + existing.model_dump.return_value["members_with_roles"] = [] + cache: Final = MagicMock() + cache.async_get_cache = AsyncMock(return_value=None) + cache.async_set_cache = AsyncMock() + cache.async_delete_cache = AsyncMock() + updated: Final = _existing_team_with_model_caps({}) + updated.metadata = {"team_member_budget_id": "member-budget-1"} + updated.model_dump.return_value.update({"metadata": updated.metadata, "members_with_roles": []}) + updated.litellm_model_table = None + update_budget_response: Final = MagicMock(budget_id="member-budget-1") + prisma: Final = MagicMock() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), + patch("litellm.proxy.proxy_server.user_api_key_cache", cache), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + patch("litellm.proxy.proxy_server.premium_user", True), + ): + prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=existing) + prisma.jsonify_team_object = lambda db_data: db_data + prisma.db.litellm_teamtable.update = AsyncMock(return_value=updated) + prisma.db.litellm_budgettable.update = AsyncMock(return_value=update_budget_response) + + await update_team( + data=UpdateTeamRequest(team_id="standalone-team-123", team_member_model_max_budget=member_caps), + http_request=MagicMock(spec=Request), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"), + ) + + budget_update: Final = prisma.db.litellm_budgettable.update.call_args.kwargs + assert budget_update["where"]["budget_id"] == "member-budget-1" + assert budget_update["data"]["budget_id"] == "member-budget-1" + stored_member_model_caps: Final = json.loads(budget_update["data"]["model_max_budget"]) + assert { + model: { + **budget, + "tpm_limit": budget.get("tpm_limit"), + "rpm_limit": budget.get("rpm_limit"), + } + for model, budget in stored_member_model_caps.items() + } == { + "gpt-4o": { + "max_budget": 5.0, + "budget_duration": "1d", + "tpm_limit": None, + "rpm_limit": None, + } + } + assert cache.async_delete_cache.await_args_list == [ + call(key="team_member_default_budget:member-budget-1"), + call(key="team_member_default_budget:member-budget-1"), + ] + + # --------------------------------------------------------------------------- # Tests for _batch_resolve_access_group_resources # --------------------------------------------------------------------------- @@ -12519,11 +12690,13 @@ async def _drive_team_write( **(existing_kwargs or {}), ) auth = user or UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="u") + user_api_key_cache: Final = MagicMock() + user_api_key_cache.async_delete_cache = AsyncMock() with ( _patch("litellm.proxy.proxy_server.prisma_client") as pc, _patch("litellm.proxy.proxy_server.llm_router", None), - _patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), + _patch("litellm.proxy.proxy_server.user_api_key_cache", user_api_key_cache), _patch("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock()), _patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), _patch( @@ -16245,13 +16418,24 @@ def test_team_model_cap_authority_skips_omitted_field_malformed_rows_and_proxy_a @pytest.mark.asyncio -async def test_new_team_persists_model_max_budget(mock_db_client, mock_admin_auth): +@pytest.mark.parametrize( + ("member_caps", "member_budget_duration"), + [ + ({"gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}}, "30d"), + ({}, None), + ], +) +async def test_new_team_persists_model_max_budget(mock_db_client, mock_admin_auth, member_caps, member_budget_duration): mock_db_client.jsonify_team_object = lambda db_data: db_data mock_db_client.get_data = AsyncMock(return_value=None) mock_db_client.update_data = AsyncMock(return_value=MagicMock()) mock_db_client.db = MagicMock() mock_db_client.db.litellm_modeltable = MagicMock() mock_db_client.db.litellm_modeltable.create = AsyncMock(return_value=MagicMock(id="model123")) + mock_db_client.db.litellm_budgettable = MagicMock() + mock_db_client.db.litellm_budgettable.create = AsyncMock( + side_effect=lambda *, data: MagicMock(budget_id=data["budget_id"]) + ) team_create_result = MagicMock(team_id="team-model-caps") team_create_result.model_dump.return_value = {"team_id": "team-model-caps"} @@ -16269,11 +16453,18 @@ async def test_new_team_persists_model_max_budget(mock_db_client, mock_admin_aut from litellm.proxy._types import NewTeamRequest from litellm.proxy.management_endpoints.team_endpoints import new_team - with patch("litellm.proxy.proxy_server.premium_user", True): # test-quality-ok: proxy_server module global is the endpoint's only injection point + member_budget_cache: Final = MagicMock() + member_budget_cache.async_delete_cache = AsyncMock() + with ( + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.user_api_key_cache", member_budget_cache), + ): await new_team( data=NewTeamRequest( team_alias="model-caps", model_max_budget={"gpt-4o": {"max_budget": 10.0, "budget_duration": "1d"}}, + team_member_budget_duration=member_budget_duration, + team_member_model_max_budget=member_caps, ), http_request=MagicMock(spec=Request), user_api_key_dict=mock_admin_auth, @@ -16283,6 +16474,75 @@ async def test_new_team_persists_model_max_budget(mock_db_client, mock_admin_aut assert team_data["model_max_budget"] == { "gpt-4o": {"max_budget": 10.0, "budget_duration": "1d", "tpm_limit": None, "rpm_limit": None} } + member_budget_row: Final = mock_db_client.db.litellm_budgettable.create.call_args.kwargs["data"] + member_budget_id: Final = member_budget_row["budget_id"] + assert team_data["metadata"]["team_member_budget_id"] == member_budget_id + assert member_budget_row.get("budget_duration") == member_budget_duration + stored_member_budget_model_caps: Final = json.loads(member_budget_row["model_max_budget"]) + expected_member_model_caps: Final = ( + { + "gpt-4o": { + "max_budget": 5.0, + "budget_duration": "1d", + "tpm_limit": None, + "rpm_limit": None, + } + } + if member_caps + else {} + ) + assert { + model: { + **budget, + "tpm_limit": budget.get("tpm_limit"), + "rpm_limit": budget.get("rpm_limit"), + } + for model, budget in stored_member_budget_model_caps.items() + } == expected_member_model_caps + member_budget_cache.async_delete_cache.assert_awaited_once_with( + key=f"team_member_default_budget:{member_budget_id}" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint_name", ["new", "update"]) +@pytest.mark.parametrize( + ("premium_user", "model_budget", "expected_status"), + [ + (False, {"gpt-4o": {"max_budget": 10.0, "budget_duration": "1d"}}, 403), + (True, {"gpt-4o": {"max_budget": 10.0}}, 400), + ], +) +async def test_team_member_model_budget_validation(endpoint_name, premium_user, model_budget, expected_status): + from fastapi import Request + + from litellm.proxy._types import NewTeamRequest, ProxyException, UpdateTeamRequest + from litellm.proxy.management_endpoints.team_endpoints import ( + new_team, + ) + from litellm.proxy.management_endpoints.team_endpoints import ( + update_team as update_team_endpoint, + ) + + endpoint: Final = new_team if endpoint_name == "new" else update_team_endpoint + data: Final = ( + NewTeamRequest(team_alias="member-model-validation", team_member_model_max_budget=model_budget) + if endpoint_name == "new" + else UpdateTeamRequest(team_id="member-model-validation", team_member_model_max_budget=model_budget) + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.premium_user", premium_user), + pytest.raises(ProxyException) as exc_info, + ): + await endpoint( + data=data, + http_request=MagicMock(spec=Request), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"), + ) + + assert exc_info.value.code == str(expected_status) @pytest.mark.asyncio @@ -17098,6 +17358,87 @@ def test_member_budget_patch_maps_temp_budget_fields() -> None: } +def test_member_budget_patch_stores_model_max_budget_as_json_and_clears_on_empty() -> None: + from litellm.proxy.management_endpoints.common_utils import member_budget_patch + + request: Final = TeamMemberUpdateRequest( + team_id="team-1", + user_id="user-1", + model_max_budget={"gpt-4o": {"budget_limit": 5, "time_period": "30d"}}, + ) + assert member_budget_patch(request) == {"model_max_budget": {"gpt-4o": {"max_budget": 5.0, "budget_duration": "30d"}}} + + cleared: Final = TeamMemberUpdateRequest(team_id="team-1", user_id="user-1", model_max_budget={}) + assert member_budget_patch(cleared) == {"model_max_budget": {}} + + +@pytest.mark.asyncio +async def test_upsert_budget_and_membership_clears_own_model_max_budget_to_empty_json() -> None: + from litellm.proxy.management_endpoints.common_utils import upsert_budget_and_membership + + own_row: Final = SimpleNamespace( + model_dump=lambda: { + "budget_id": "member-b", + "max_budget": 50.0, + "model_max_budget": {"gpt-4o": {"max_budget": 2.5, "budget_duration": "7d"}}, + } + ) + tx: Final = MagicMock() + tx.litellm_budgettable.find_unique = AsyncMock(return_value=own_row) + tx.litellm_budgettable.update = AsyncMock() + tx.litellm_teammembership.update = AsyncMock() + + await upsert_budget_and_membership( + tx, + team_id="team-1", + user_id="user-1", + existing_budget_id="member-b", + user_api_key_dict=UserAPIKeyAuth(user_id="admin"), + budget_patch={"model_max_budget": None}, + team_default_budget_id="default-b", + ) + + assert tx.litellm_budgettable.update.await_args.kwargs["data"]["model_max_budget"] == "{}" + tx.litellm_teammembership.update.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_upsert_budget_and_membership_clone_keeps_shared_default_model_max_budget_as_json() -> None: + from litellm.proxy.management_endpoints.common_utils import upsert_budget_and_membership + + default_model_budget: Final = {"gpt-4o": {"max_budget": 5.0, "budget_duration": "30d"}} + default_row: Final = SimpleNamespace( + model_dump=lambda: { + "budget_id": "default-b", + "max_budget": 10.0, + "model_max_budget": default_model_budget, + "budget_reset_at": None, + } + ) + tx: Final = MagicMock() + tx.litellm_budgettable.find_unique = AsyncMock(return_value=default_row) + tx.litellm_budgettable.create = AsyncMock(return_value=SimpleNamespace(budget_id="member-b")) + tx.litellm_teammembership.upsert = AsyncMock() + + await upsert_budget_and_membership( + tx, + team_id="team-1", + user_id="user-1", + existing_budget_id="default-b", + user_api_key_dict=UserAPIKeyAuth(user_id="admin"), + budget_patch={"tpm_limit": 7}, + team_default_budget_id="default-b", + ) + + create_data: Final = tx.litellm_budgettable.create.await_args.kwargs["data"] + assert create_data["tpm_limit"] == 7 + assert create_data["max_budget"] == 10.0 + assert json.loads(create_data["model_max_budget"]) == {} + assert tx.litellm_teammembership.upsert.await_args.kwargs["data"]["update"] == { + "litellm_budget_table": {"connect": {"budget_id": "member-b"}} + } + + def test_team_member_update_request_temp_budget_fields_must_be_set_together() -> None: with pytest.raises(ValidationError, match="temp_budget_increase and temp_budget_expiry must be set together"): TeamMemberUpdateRequest(team_id="team-1", user_id="user-1", temp_budget_increase=50.0) @@ -17211,3 +17552,18 @@ def test_aggregated_date_range_error_accepts_canonical_dates_and_keeps_range_che assert aggregated_date_range_error("2026-09-24", "2026-09-26") is None assert aggregated_date_range_error("2026-09-26", "2026-09-24") == "end_date must be on or after start_date" assert aggregated_date_range_error("2020-01-01", "2026-12-31") == "Date range must be at most 400 days" + + +@pytest.mark.asyncio +async def test_default_budget_cache_eviction_does_not_fail_after_redis_error() -> None: + from litellm.proxy.management_endpoints.team_endpoints import TeamMemberBudgetHandler + + user_api_key_cache = MagicMock() + user_api_key_cache.async_delete_cache = AsyncMock(side_effect=RuntimeError("redis unavailable")) + + await TeamMemberBudgetHandler._evict_default_budget_cache( + budget_id="budget_456", + user_api_key_cache=user_api_key_cache, + ) + + user_api_key_cache.async_delete_cache.assert_awaited_once_with(key="team_member_default_budget:budget_456") diff --git a/tests/unit/proxy/management_helpers/test_management_helpers_utils.py b/tests/unit/proxy/management_helpers/test_management_helpers_utils.py index b450a4dfd02..644f0656c7a 100644 --- a/tests/unit/proxy/management_helpers/test_management_helpers_utils.py +++ b/tests/unit/proxy/management_helpers/test_management_helpers_utils.py @@ -1,3 +1,4 @@ +import json from collections.abc import Mapping, Sequence from datetime import datetime, timezone from types import SimpleNamespace @@ -817,6 +818,10 @@ async def test_add_new_member_with_user_email_links_default_budget(): mock_prisma_client.db.litellm_budgettable.create.assert_not_called() +def _as_stored(data: Mapping[str, object]) -> dict[str, object]: + return {k: json.loads(v) if k == "model_max_budget" and isinstance(v, str) else v for k, v in data.items()} + + class _FakeBudgetTable: def __init__(self) -> None: self.rows: dict[str, dict[str, object]] = {} @@ -829,14 +834,14 @@ class _FakeBudgetTable: self, *, data: Mapping[str, object], include: Mapping[str, bool] | None = None ) -> LiteLLM_BudgetTable: budget_id: Final = str(data.get("budget_id") or uuid.uuid4()) - self.rows[budget_id] = {**data, "budget_id": budget_id} + self.rows[budget_id] = {**_as_stored(data), "budget_id": budget_id} return self._record(budget_id) async def find_unique(self, *, where: Mapping[str, str]) -> LiteLLM_BudgetTable | None: return self._record(where["budget_id"]) if where["budget_id"] in self.rows else None async def update(self, *, where: Mapping[str, str], data: Mapping[str, object]) -> LiteLLM_BudgetTable: - self.rows[where["budget_id"]] = {**self.rows[where["budget_id"]], **data} + self.rows[where["budget_id"]] = {**self.rows[where["budget_id"]], **_as_stored(data)} return self._record(where["budget_id"]) diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index fb8d98ef61e..bd66bc24837 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -7918,14 +7918,17 @@ def test_passthrough_carries_the_per_model_budgets(): key_budget = {"claude-opus-4-8": {"budget_limit": 1.0, "time_period": "18h"}} user_budget = {"claude-opus-4-8": {"budget_limit": 2.0, "time_period": "1mo"}} end_user_budget = {"claude-opus-4-8": {"budget_limit": 3.0, "time_period": "1d"}} + team_member_budget = {"claude-opus-4-8": {"budget_limit": 4.0, "time_period": "1d"}} kwargs = _passthrough_kwargs_for_reservation( UserAPIKeyAuth( token="hash", user_id="u-1", + team_id="team-1", model_max_budget=key_budget, user_model_max_budget=user_budget, end_user_model_max_budget=end_user_budget, + team_member_model_max_budget=team_member_budget, ) ) @@ -7933,6 +7936,7 @@ def test_passthrough_carries_the_per_model_budgets(): assert metadata["user_api_key_model_max_budget"] == key_budget assert metadata["user_api_key_user_model_max_budget"] == user_budget assert metadata["user_api_key_end_user_model_max_budget"] == end_user_budget + assert metadata["user_api_key_team_member_model_max_budget"] == team_member_budget def test_passthrough_budget_metadata_cannot_be_forged_by_the_request_body(): @@ -7981,7 +7985,10 @@ async def test_user_defined_passthrough_is_neither_tracked_nor_enforced(metadata budget: Final = {"managed-model": {"budget_limit": 0.1, "time_period": "1d"}} limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(DualCache()) auth: Final = UserAPIKeyAuth( - api_key="custom-key", token="custom-key", team_id="shared-team", team_model_max_budget=budget, + api_key="custom-key", + token="custom-key", + team_id="shared-team", + team_model_max_budget=budget, ) endpoint: Final = create_pass_through_route( endpoint="/custom-budget-test", diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index b040cc861bf..0f66674daf9 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -17,6 +17,7 @@ from fastapi.responses import JSONResponse, StreamingResponse import litellm from litellm._uuid import uuid from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame, anthropic_error_sse_frame +from litellm.caching.caching import DualCache from litellm.litellm_core_utils.bug_report import ( DISABLE_ENV_VAR, ISSUE_URL_BASE, @@ -59,11 +60,18 @@ from litellm.proxy.common_request_processing import ( sse_error_payload, ) from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header +from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError from litellm.proxy.common_utils.sse_keepalive import ANTHROPIC_PING_SSE_CHUNK from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.proxy._types import Litellm_EntityType, ProxyErrorTypes, ProxyException from litellm.proxy._types import UserAPIKeyAuth as ProxyUserAPIKeyAuth +from litellm.proxy.auth.fallback_budget import RouterFallbackBudgetCheck +from litellm.proxy.hooks.model_max_budget_limiter import ( + PROXY_VirtualKeyModelMaxBudgetLimiter, + model_budget_spend_cache_key, + team_member_budget_entity_id, +) from litellm.proxy.utils import ProxyLogging from litellm.router import Router from litellm.router_utils.add_retry_fallback_headers import prepare_response_for_header_attachment @@ -7097,6 +7105,9 @@ class TestPreCallWithFallbacksOnLocalRateLimit: fallbacks: list[dict[str, list[str]]], model_guardrails: dict[str, list[str]] | None = None, untagged_rate_limited_models: frozenset[str] = frozenset(), + fallback_budget_check: RouterFallbackBudgetCheck | None = None, + free_models: frozenset[str] = frozenset(), + extra_models: frozenset[str] = frozenset(), ) -> tuple[ProxyLogging, litellm.Router, ProxyConfig, list[str]]: """Real v3 limiter (the default ``parallel_request_limiter``) wired in through the ``proxy_logging_obj`` seam, so ``common_processing_pre_call_logic`` runs for real: @@ -7145,11 +7156,17 @@ class TestPreCallWithFallbacksOnLocalRateLimit: "api_key": "fake", **({"guardrails": guardrails_by_group[group]} if group in guardrails_by_group else {}), }, + **( + {"model_info": {"input_cost_per_token": 0, "output_cost_per_token": 0}} + if group in free_models + else {} + ), } for chain in fallbacks - for group in (*chain.keys(), *(m for models in chain.values() for m in models)) + for group in (*chain.keys(), *(m for models in chain.values() for m in models), *sorted(extra_models)) ], fallbacks=fallbacks, + fallback_budget_check=fallback_budget_check, ) return proxy_logging_obj, router, proxy_server.ProxyConfig(), limiter_models @@ -7636,6 +7653,138 @@ class TestPreCallWithFallbacksOnLocalRateLimit: assert data["metadata"]["guardrails"] == [structured_guardrail] assert rig[3] == [primary_model, primary_model, fallback_model] + @staticmethod + async def _install_member_spend(monkeypatch: pytest.MonkeyPatch, model: str, spend: float) -> None: + from litellm.proxy import proxy_server + + member_limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache()) + await member_limiter.dual_cache.async_set_cache( + key=model_budget_spend_cache_key( + entity_type=Litellm_EntityType.TEAM_MEMBER, + entity_id=team_member_budget_entity_id(user_id="user-1", team_id="team-1"), + budget_model=model, + budget_duration="1d", + ), + value=spend, + ) + monkeypatch.setattr(proxy_server, "model_max_budget_limiter", member_limiter) + + def _member_key(self, primary_model: str, fallback_model: str) -> ProxyUserAPIKeyAuth: + return self._otel_key(model_rpm_limit={primary_model: 1}).model_copy( + update={ + "user_id": "user-1", + "team_id": "team-1", + "team_member_model_max_budget": {fallback_model: {"max_budget": 5.0, "budget_duration": "1d"}}, + } + ) + + @staticmethod + def _client_request(model: str) -> dict[str, object]: + return {"model": model, "messages": [{"role": "user", "content": "hi"}]} + + @pytest.mark.asyncio + async def test_rate_limit_fallback_skips_backup_over_member_model_budget( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + primary_model: Final = "gpt-4.1" + fallback_model: Final = "gpt-4.1-mini" + await self._install_member_spend(monkeypatch, fallback_model, spend=5.0) + key: Final = self._member_key(primary_model, fallback_model) + rig: Final = self._v3_limiter_rig(monkeypatch, key, [{primary_model: [fallback_model]}]) + + _, (first_data, _) = await self._pre_call(self._client_request(primary_model), key, rig) + processor: Final = ProxyBaseLLMRequestProcessing(data=self._client_request(primary_model)) + with pytest.raises(ProxyRateLimitError): + await self._run(processor, key, rig) + + assert first_data["model"] == primary_model + assert rig[3] == [primary_model, primary_model] + + @pytest.mark.asyncio + async def test_rate_limit_fallback_skips_backup_aliased_to_model_over_member_budget( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + primary_model: Final = "gpt-4.1" + fallback_model: Final = "gpt-4.1-mini" + resolved_model: Final = "expensive" + await self._install_member_spend(monkeypatch, resolved_model, spend=5.0) + key: Final = self._member_key(primary_model, fallback_model).model_copy( + update={ + "router_settings": {"model_group_alias": {fallback_model: resolved_model}}, + "team_member_model_max_budget": {resolved_model: {"max_budget": 5.0, "budget_duration": "1d"}}, + } + ) + rig: Final = self._v3_limiter_rig( + monkeypatch, + key, + [{primary_model: [fallback_model]}], + extra_models=frozenset({resolved_model}), + ) + + await self._pre_call(self._client_request(primary_model), key, rig) + processor: Final = ProxyBaseLLMRequestProcessing(data=self._client_request(primary_model)) + with pytest.raises(ProxyRateLimitError): + await self._run(processor, key, rig) + + assert processor.data["model"] == primary_model + assert rig[3] == [primary_model, primary_model] + + @pytest.mark.asyncio + async def test_rate_limit_fallback_uses_backup_within_member_model_budget( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + primary_model: Final = "gpt-4.1" + fallback_model: Final = "gpt-4.1-mini" + await self._install_member_spend(monkeypatch, fallback_model, spend=4.0) + key: Final = self._member_key(primary_model, fallback_model) + rig: Final = self._v3_limiter_rig(monkeypatch, key, [{primary_model: [fallback_model]}]) + + await self._pre_call(self._client_request(primary_model), key, rig) + _, (data, logging_obj) = await self._pre_call(self._client_request(primary_model), key, rig) + + assert data["model"] == fallback_model + assert logging_obj.model == fallback_model + assert rig[3] == [primary_model, primary_model, fallback_model] + + @pytest.mark.asyncio + async def test_rate_limit_fallback_uses_free_backup_with_spent_member_model_budget( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + primary_model: Final = "gpt-4.1" + fallback_model: Final = "gpt-4.1-mini" + await self._install_member_spend(monkeypatch, fallback_model, spend=5.0) + key: Final = self._member_key(primary_model, fallback_model) + rig: Final = self._v3_limiter_rig( + monkeypatch, key, [{primary_model: [fallback_model]}], free_models=frozenset({fallback_model}) + ) + + await self._pre_call(self._client_request(primary_model), key, rig) + _, (data, _) = await self._pre_call(self._client_request(primary_model), key, rig) + + assert data["model"] == fallback_model + assert rig[3] == [primary_model, primary_model, fallback_model] + + @pytest.mark.asyncio + async def test_rate_limit_fallback_ignores_key_budget_already_admitted_at_auth( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + primary_model: Final = "gpt-4.1" + fallback_model: Final = "gpt-4.1-mini" + key: Final = self._otel_key(model_rpm_limit={primary_model: 1}).model_copy( + update={"token": "hashed-key", "max_budget": 1.0, "spend": 2.0} + ) + rig: Final = self._v3_limiter_rig( + monkeypatch, + key, + [{primary_model: [fallback_model]}], + fallback_budget_check=RouterFallbackBudgetCheck(is_enforced=lambda: True), + ) + + await self._pre_call(self._client_request(primary_model), key, rig) + _, (data, _) = await self._pre_call(self._client_request(primary_model), key, rig) + + assert data["model"] == fallback_model + assert rig[3] == [primary_model, primary_model, fallback_model] class _RecordingSuccessLogger(CustomLogger): def __init__(self): diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index cf04b9ff91e..3a6aac9be05 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -2016,6 +2016,8 @@ async def test_add_litellm_data_to_request_user_spend_and_budget(): api_key="hashed-key", metadata={}, team_metadata={}, + team_id="team-1", + team_member_model_max_budget={"gpt-3.5-turbo": {"max_budget": 15.0, "budget_duration": "1d"}}, user_spend=150.0, user_max_budget=500.0, ) @@ -2032,6 +2034,9 @@ async def test_add_litellm_data_to_request_user_spend_and_budget(): metadata = updated_data.get("metadata", {}) assert metadata["user_api_key_user_spend"] == 150.0 assert metadata["user_api_key_user_max_budget"] == 500.0 + assert metadata["user_api_key_team_member_model_max_budget"] == { + "gpt-3.5-turbo": {"max_budget": 15.0, "budget_duration": "1d"} + } @pytest.mark.asyncio diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index e0829e42557..3694ecea4ed 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -14,6 +14,7 @@ from litellm.router_utils.common_utils import ( filter_web_search_deployments, provider_for_generic_call, resolve_model_group_alias, + resolve_served_model, truncate_fallback_error_detail, PROVIDER_SCOPED_CREDENTIAL_PARAMS, warn_on_provider_credential_mismatch, @@ -566,6 +567,25 @@ class TestResolveModelGroupAlias: assert router.get_model_from_alias("group-b") is None +@pytest.mark.parametrize( + "model, model_alias_map, key_aliases, model_group_alias, expected", + [ + pytest.param("a", {"a": "b"}, {"b": "c"}, {"c": "d"}, "d", id="global-key-router-order"), + pytest.param("a", {}, ["a", "b"], {}, "a", id="non-mapping-key-aliases-ignored"), + pytest.param("a", {}, {}, {"a": {"model": "x"}}, "x", id="router-item-entry"), + pytest.param("a", {}, {}, {}, "a", id="no-alias"), + ], +) +def test_resolves_served_model_aliases_in_order( + model: str, + model_alias_map: object, + key_aliases: object, + model_group_alias: object, + expected: str, +) -> None: + assert resolve_served_model(model, model_alias_map, key_aliases, model_group_alias) == expected + + class TestTruncateFallbackErrorDetail: def test_short_detail_is_returned_unchanged(self): assert truncate_fallback_error_detail("boom") == "boom" diff --git a/ui/litellm-dashboard/src/components/Teams.integration.test.tsx b/ui/litellm-dashboard/src/components/Teams.integration.test.tsx new file mode 100644 index 00000000000..a6801e3c996 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Teams.integration.test.tsx @@ -0,0 +1,83 @@ +import { fireEvent, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { renderWithProviders, testQueryClient } from "../../tests/test-utils"; +import Teams from "./Teams"; + +const sessionCookie = () => { + const encode = (value: object) => + btoa(JSON.stringify(value)).replaceAll("=", "").replaceAll("+", "-").replaceAll("/", "_"); + const claims = { + key: "sk-session-test", + user_id: "test-admin", + user_role: "proxy_admin", + premium_user: true, + exp: Date.now() / 1000 + 3600, + }; + document.cookie = `token=${encode({ alg: "none" })}.${encode(claims)}.test; Path=/`; +}; + +const responseFor = (url: string): object | object[] => { + if (url.includes("/v2/team/list")) return { teams: [], total: 0, page: 1, page_size: 50, total_pages: 1 }; + if (url.includes("/models")) return { data: [{ id: "gpt-4" }] }; + if (url.includes("/v1/access_group")) return []; + if (url.includes("/v1/mcp/access_groups")) return { access_groups: [] }; + if (url.includes("/guardrails/list")) return { guardrails: [] }; + if (url.includes("/policies/list")) return { policies: [] }; + if (url.includes("/organization/list")) return { organizations: [] }; + if (url.includes("/team/new")) return { team_id: "new-team-1" }; + if (url.includes("/default")) return { values: {} }; + if (url.includes("/litellm-ui-config")) return { server_root_path: "", proxy_base_url: "" }; + return {}; +}; + +beforeEach(() => { + testQueryClient.clear(); + sessionCookie(); + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL) => Response.json(responseFor(String(input)))), + ); +}); + +afterEach(() => { + testQueryClient.clear(); + vi.unstubAllGlobals(); + document.cookie = "token=; Max-Age=0; Path=/"; +}); + +describe("Teams creation integration", () => { + it("sends a configured default member per-model budget", async () => { + const user = userEvent.setup({ delay: null }); + renderWithProviders(); + + await user.click(screen.getAllByRole("button", { name: /create team/i })[0]); + await screen.findByLabelText(/team name/i); + await user.click(screen.getByText("Additional Settings")); + await screen.findByText(/Team Member Key Duration/); + + const addBudgetButtons = screen.getAllByRole("button", { name: /Add Model Budget/i }); + await user.click(addBudgetButtons[1]); + const modelSelectors = screen.getAllByPlaceholderText("Select model"); + await user.click(modelSelectors[0]); + await user.click(await screen.findByRole("option", { name: "gpt-4" })); + fireEvent.change(screen.getAllByPlaceholderText("Max spend ($)")[0], { target: { value: "4" } }); + await user.click(screen.getByText("Additional Settings")); + + fireEvent.change(screen.getByLabelText(/team name/i), { target: { value: "Budget Team" } }); + await user.click(screen.getAllByRole("button", { name: /create team/i }).at(-1)!); + await waitFor(() => { + expect(vi.mocked(fetch)).toHaveBeenCalled(); + }); + + const createRequest = vi.mocked(fetch).mock.calls.find(([input]) => String(input).includes("/team/new")); + expect(createRequest).toBeDefined(); + const payload = JSON.parse(String(createRequest?.[1]?.body)) as Record; + expect(payload.team_member_model_max_budget).toStrictEqual({ + "gpt-4": { budget_limit: 4, time_period: "30d" }, + }); + expect(JSON.parse(JSON.stringify(payload)).team_member_model_max_budget).toStrictEqual({ + "gpt-4": { budget_limit: 4, time_period: "30d" }, + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Teams.tsx b/ui/litellm-dashboard/src/components/Teams.tsx index 0b773fd0f04..9971010c3ef 100644 --- a/ui/litellm-dashboard/src/components/Teams.tsx +++ b/ui/litellm-dashboard/src/components/Teams.tsx @@ -51,6 +51,7 @@ import BudgetDurationDropdown, { import { Organization, getDefaultTeamSettings, getGuardrailsList, getPoliciesList, teamDeleteCall } from "./networking"; import NumericalInput from "./shared/numerical_input"; import { ModelMaxBudget, ModelMaxBudgetField } from "./key_team_helpers/ModelMaxBudgetEditor"; +import { modelMaxBudgetUpdate } from "./key_team_helpers/modelMaxBudgetPayload"; import VectorStoreSelector from "./vector_store_management/VectorStoreSelector"; import SearchToolSelector from "./search_tools/SearchToolSelector"; import SkillSelector from "./skills/SkillSelector"; @@ -280,6 +281,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, isViewOnly const [loggingSettings, setLoggingSettings] = useState([]); const [modelAliases, setModelAliases] = useState<{ [key: string]: string }>({}); const [modelMaxBudget, setModelMaxBudget] = useState({}); + const [teamMemberModelMaxBudget, setTeamMemberModelMaxBudget] = useState({}); const [routerSettings, setRouterSettings] = useState(null); const [routerSettingsKey, setRouterSettingsKey] = useState(0); @@ -358,6 +360,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, isViewOnly setLoggingSettings([]); setModelAliases({}); setModelMaxBudget({}); + setTeamMemberModelMaxBudget({}); setRouterSettings(null); setRouterSettingsKey((prev) => prev + 1); }; @@ -551,9 +554,11 @@ const Teams: React.FC = ({ accessToken, userID, userRole, isViewOnly } const { require_trace_id: requireTraceId, ...teamCreateValues } = formValues; + const teamMemberModelBudgets = modelMaxBudgetUpdate(teamMemberModelMaxBudget, {}); await teamCreateCall(accessToken, { ...teamCreateValues, ...(requireTraceId === undefined ? {} : { require_trace_id: requireTraceId }), + ...(teamMemberModelBudgets !== undefined ? { team_member_model_max_budget: teamMemberModelBudgets } : {}), models: normalizeTeamModelSelection(formValues.models), }); toast.success("Team created"); @@ -912,6 +917,15 @@ const Teams: React.FC = ({ accessToken, userID, userRole, isViewOnly /> )} + (() => modelMaxBudgetToEntries(value)); + const incoming = JSON.stringify(value ?? {}); + const [syncedValue, setSyncedValue] = useState(incoming); + if (incoming !== syncedValue) { + setSyncedValue(incoming); + if (incoming !== JSON.stringify(entriesToModelMaxBudget(entries))) setEntries(modelMaxBudgetToEntries(value)); + } + const emitChange = (updated: ModelBudgetEntry[]) => { setEntries(updated); onChange(entriesToModelMaxBudget(updated)); @@ -219,14 +226,15 @@ export function ModelMaxBudgetEditor({ interface ModelMaxBudgetFieldProps extends ModelMaxBudgetEditorProps { hint: string; + label?: string; } /** The editor with its label, so every form that offers it presents it the same way. */ -export function ModelMaxBudgetField({ hint, ...editorProps }: ModelMaxBudgetFieldProps) { +export function ModelMaxBudgetField({ hint, label = "Per-Model Budgets", ...editorProps }: ModelMaxBudgetFieldProps) { return ( - Per-Model Budgets + {label} diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 54b2066c9c5..0fbf3d36355 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -94,6 +94,7 @@ import { Team } from "./key_team_helpers/key_list"; import { EmailEventSettingsResponse, EmailEventSettingsUpdateRequest } from "./email_events/types"; import type { ListPluginsResponse, SkillRegisterRequest } from "./claude_code_plugins/types"; import type { ModelBudgetUsage, ModelMaxBudget } from "./key_team_helpers/ModelMaxBudgetEditor"; +import type { StoredModelMaxBudget } from "./key_team_helpers/modelMaxBudgetPayload"; import type { ObjectPermission } from "./object_permission_types"; import type { components } from "@/lib/http/schema"; import { fetchClient } from "@/lib/http/api"; @@ -2831,6 +2832,7 @@ export interface Member { allowed_models?: string[] | null; temp_budget_increase?: number | null; temp_budget_expiry?: string | null; + model_max_budget?: StoredModelMaxBudget | null; } export const teamMemberAddCall = async (accessToken: string, teamId: string, formValues: Member) => { @@ -2971,6 +2973,9 @@ export const teamMemberUpdateCall = async ( if ("temp_budget_expiry" in formValues) { requestBody.temp_budget_expiry = orNull(formValues.temp_budget_expiry); } + if (formValues.model_max_budget !== undefined) { + requestBody.model_max_budget = formValues.model_max_budget; + } const response = await fetch(url, { method: "POST", diff --git a/ui/litellm-dashboard/src/components/team/EditMembership.integration.test.tsx b/ui/litellm-dashboard/src/components/team/EditMembership.integration.test.tsx index 45800a18ff8..2c98d17f961 100644 --- a/ui/litellm-dashboard/src/components/team/EditMembership.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/team/EditMembership.integration.test.tsx @@ -1,4 +1,6 @@ +import { useState } from "react"; import { fireEvent, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders } from "../../../tests/test-utils"; import EditMembership from "./EditMembership"; @@ -104,6 +106,79 @@ describe("EditMembership submit payload", () => { }); }); + it("seeds the per-model budget editor from the member's stored budget and submits the edited cap", async () => { + const modelBudgetConfig = { + ...teamMemberConfig, + additionalFields: [ + ...additionalFields, + { + name: "model_max_budget", + label: "Per-Model Budgets", + type: "model-max-budget" as const, + availableModels: ["gpt-4o"], + premiumUser: true, + }, + ], + }; + const member = { + user_id: "u1", + user_email: "a@b.com", + role: "user", + model_max_budget: { "gpt-4o": { max_budget: 5, budget_duration: "30d" } }, + }; + renderEdit(modelBudgetConfig, member); + + const cap = await screen.findByPlaceholderText("Max spend ($)"); + expect(cap).toHaveValue(5); + fireEvent.change(cap, { target: { value: "9" } }); + save(); + + await waitFor(() => expect(onSubmit).toHaveBeenCalledOnce()); + expect(submitted().model_max_budget).toStrictEqual({ "gpt-4o": { budget_limit: 9, time_period: "30d" } }); + }); + + it("saves a role-only edit for a member with stored model caps", async () => { + const modelBudgetConfig = { + ...teamMemberConfig, + additionalFields: [ + ...additionalFields, + { + name: "model_max_budget", + label: "Per-Model Budgets", + type: "model-max-budget" as const, + availableModels: ["gpt-4o"], + premiumUser: true, + }, + ], + }; + const member = { + user_id: "u1", + user_email: "a@b.com", + role: "user", + model_max_budget: { "gpt-4o": { max_budget: 5, budget_duration: "30d" } }, + }; + renderEdit(modelBudgetConfig, member); + + const user = userEvent.setup(); + await user.click(screen.getByRole("combobox", { name: "Role(Current: User)" })); + await user.click(await screen.findByRole("option", { name: "Admin" })); + save(); + + await waitFor(() => expect(onSubmit).toHaveBeenCalledOnce()); + const expectedPayload = { + user_email: "a@b.com", + user_id: "u1", + role: "admin", + max_budget_in_team: null, + budget_duration: null, + tpm_limit: null, + rpm_limit: null, + allowed_models: [], + model_max_budget: { "gpt-4o": { budget_limit: 5, time_period: "30d" } }, + }; + expect(submitted()).toStrictEqual(expectedPayload); + }); + it("omits a field the config hides even when the member record carries it", async () => { renderEdit({ ...orgMemberConfig, showUserId: false }, { user_id: "u1", user_email: "a@b.com", role: "admin" }); @@ -342,3 +417,57 @@ describe("EditMembership submit payload", () => { expect(screen.getByLabelText("Email")).toHaveValue(""); }); }); + +it("discards an unfinished per-model budget row when an edit session is reopened", async () => { + const user = userEvent.setup({ delay: null }); + const modelBudgetConfig = { + ...teamMemberConfig, + additionalFields: [ + ...additionalFields, + { + name: "model_max_budget", + label: "Per-Model Budgets", + type: "model-max-budget" as const, + availableModels: ["gpt-4o"], + premiumUser: true, + }, + ], + }; + const member = { user_id: "u1", user_email: "a@b.com", role: "user", model_max_budget: {} }; + const otherMember = { ...member, user_id: "u2", user_email: "c@d.com" }; + const Harness = ({ initialData }: { initialData: typeof member }) => { + const [visible, setVisible] = useState(false); + return ( + <> + + setVisible(false)} + onSubmit={vi.fn()} + mode="edit" + config={modelBudgetConfig as never} + initialData={initialData as never} + /> + + ); + }; + + const { rerender } = renderWithProviders(); + + await user.click(screen.getByRole("button", { name: "Open Edit Member" })); + await user.click(screen.getByRole("button", { name: /Add Model Budget/i })); + expect(screen.getByPlaceholderText("Select model")).toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Cancel" })); + await user.click(screen.getByRole("button", { name: "Open Edit Member" })); + + expect(screen.queryByPlaceholderText("Select model")).not.toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: /Add Model Budget/i })); + expect(screen.getByPlaceholderText("Select model")).toBeInTheDocument(); + + rerender(); + expect(screen.queryByPlaceholderText("Select model")).not.toBeInTheDocument(); +}); diff --git a/ui/litellm-dashboard/src/components/team/EditMembership.tsx b/ui/litellm-dashboard/src/components/team/EditMembership.tsx index 4aaeb42da98..4e1fa4cfbcf 100644 --- a/ui/litellm-dashboard/src/components/team/EditMembership.tsx +++ b/ui/litellm-dashboard/src/components/team/EditMembership.tsx @@ -12,11 +12,13 @@ import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { UtcDateTimeInput } from "@/components/shared/form/UtcDateTimeInput"; +import { ModelMaxBudgetEditor } from "../key_team_helpers/ModelMaxBudgetEditor"; import { useZodForm } from "@/lib/forms/useZodForm"; import { buildMemberFormData, buildMemberFormValues, emptyMemberFormValues, + isModelMaxBudget, TEMP_BUDGET_PAIR_MESSAGE, tempBudgetPairError, type MemberAdditionalField, @@ -51,7 +53,11 @@ const ROLE_REQUIRED_MESSAGE = "Please select a role!"; const isEmailish = (value: string): boolean => value === "" || z.email({ pattern: z.regexes.html5Email }).safeParse(value).success; -const memberFieldSchema = z.union([z.string(), z.number(), z.null(), z.array(z.string())]).optional(); +const modelMaxBudgetSchema = z.record(z.string(), z.looseObject({ budget_limit: z.number(), time_period: z.string() })); + +const memberFieldSchema = z + .union([z.string(), z.number(), z.null(), z.array(z.string()), modelMaxBudgetSchema]) + .optional(); const buildMemberSchema = (config: ModalConfig): z.ZodType => { const shape = { @@ -171,6 +177,16 @@ const MemberModal = ({ onChange={(next) => onChange(mode === "add" ? next ?? undefined : next)} /> ); + case "model-max-budget": + return ( + + ); case "utc-datetime": return ( { expect(screen.queryByText("Reset member budgets?")).not.toBeInTheDocument(); expect(bulkUpdatePOST).not.toHaveBeenCalled(); }); + + it("omits the default member model budget when it has not changed", async () => { + const user = userEvent.setup({ delay: null }); + const storedBudget = { "gpt-4": { budget_limit: 3, time_period: "30d" } }; + const data = createMockTeamData({ + models: ["gpt-4"], + team_member_budget_table: { + max_budget: 10, + budget_duration: null, + tpm_limit: null, + rpm_limit: null, + model_max_budget: storedBudget, + }, + }); + vi.mocked(networking.teamInfoCall).mockResolvedValue(data); + vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: savedTeam, team_id: "123" }); + + renderWithProviders(); + await waitFor(() => expect(screen.queryAllByText("Test Team").length).toBeGreaterThan(0)); + await user.click(screen.getByRole("tab", { name: "Settings" })); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + await user.click(screen.getByText("Team Member Settings")); + await screen.findByPlaceholderText("Max spend ($)"); + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(networking.teamUpdateCall).toHaveBeenCalled()); + expect(vi.mocked(networking.teamUpdateCall).mock.calls[0][1]).not.toHaveProperty("team_member_model_max_budget"); + }); + + it("sends the default member model budget after an edit", async () => { + const user = userEvent.setup({ delay: null }); + const storedBudget = { "gpt-4": { budget_limit: 3, time_period: "30d" } }; + const data = createMockTeamData({ + models: ["gpt-4"], + team_member_budget_table: { + max_budget: 10, + budget_duration: null, + tpm_limit: null, + rpm_limit: null, + model_max_budget: storedBudget, + }, + }); + vi.mocked(networking.teamInfoCall).mockResolvedValue(data); + vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: savedTeam, team_id: "123" }); + + renderWithProviders(); + await waitFor(() => expect(screen.queryAllByText("Test Team").length).toBeGreaterThan(0)); + await user.click(screen.getByRole("tab", { name: "Settings" })); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + await user.click(screen.getByText("Team Member Settings")); + fireEvent.change(await screen.findByPlaceholderText("Max spend ($)"), { target: { value: "7" } }); + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(networking.teamUpdateCall).toHaveBeenCalled()); + expect(vi.mocked(networking.teamUpdateCall).mock.calls[0][1].team_member_model_max_budget).toStrictEqual({ + "gpt-4": { budget_limit: 7, time_period: "30d" }, + }); + }); + + it("offers access-group models in the default member budget picker", async () => { + const user = userEvent.setup({ delay: null }); + vi.mocked(networking.teamInfoCall).mockResolvedValue( + createMockTeamData({ + models: ["gpt-4o"], + access_group_models: ["claude-haiku-4-5"], + }) as TeamData, + ); + + renderWithProviders(); + await waitFor(() => expect(screen.queryAllByText("Test Team").length).toBeGreaterThan(0)); + await user.click(screen.getByRole("tab", { name: "Settings" })); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + await user.click(screen.getByText("Team Member Settings")); + await user.click(screen.getAllByRole("button", { name: /add model budget/i })[0]); + await user.click(screen.getByPlaceholderText("Select model")); + + expect(await screen.findByRole("option", { name: "claude-haiku-4-5" })).toBeInTheDocument(); + }); + + it("offers user models in the default member budget picker when the team model list is empty", async () => { + const user = userEvent.setup({ delay: null }); + vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData({ models: [] }) as TeamData); + + renderWithProviders(); + await waitFor(() => expect(screen.queryAllByText("Test Team").length).toBeGreaterThan(0)); + await user.click(screen.getByRole("tab", { name: "Settings" })); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + await user.click(screen.getByText("Team Member Settings")); + await user.click(screen.getAllByRole("button", { name: /add model budget/i })[0]); + await user.click(screen.getByPlaceholderText("Select model")); + + expect(await screen.findByRole("option", { name: "gpt-4o" })).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "claude-haiku-4-5" })).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index a782112edb2..7663ef8c588 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -286,7 +286,7 @@ export interface TeamMembership { max_parallel_requests: number | null; tpm_limit: number | null; rpm_limit: number | null; - model_max_budget: Record | null; + model_max_budget: StoredModelMaxBudget | null; budget_duration: string | null; budget_reset_at: string | null; allowed_models?: string[] | null; @@ -339,6 +339,7 @@ export interface TeamData { budget_duration: string | null; tpm_limit: number | null; rpm_limit: number | null; + model_max_budget?: StoredModelMaxBudget | null; } | null; }; keys: any[]; @@ -631,6 +632,7 @@ const TeamInfoView: React.FC = ({ const [isTeamSaving, setIsTeamSaving] = useState(false); const [teamModelAliases, setTeamModelAliases] = useState>({}); const [teamModelMaxBudget, setTeamModelMaxBudget] = useState({}); + const [teamMemberModelMaxBudget, setTeamMemberModelMaxBudget] = useState({}); const routerSettingsRef = React.useRef(null); const [organization, setOrganization] = useState(null); const { userRole, isViewOnly } = useAuthorized(); @@ -661,12 +663,20 @@ const TeamInfoView: React.FC = ({ ] as const ).find(([failed]) => failed)?.[1] ?? null; const availableRateLimitModels = useMemo(() => { - const selected = watchedModels ?? teamData?.team_info?.models ?? []; + const selected = (isEditing ? watchedModels : undefined) ?? teamData?.team_info?.models ?? []; if (selected.includes("all-proxy-models") || selected.includes("all-team-models")) { return userModels; } return unfurlWildcardModelsInList(selected, userModels); - }, [watchedModels, teamData, userModels]); + }, [isEditing, watchedModels, teamData, userModels]); + const availableModelBudgetModels = useMemo(() => { + const selected = (isEditing ? watchedModels : undefined) ?? teamData?.team_info?.models ?? []; + if (selected.length === 0 || selected.includes("all-proxy-models") || selected.includes("all-team-models")) { + return userModels; + } + const granted = [...selected, ...(teamData?.team_info?.access_group_models ?? [])]; + return unfurlWildcardModelsInList(Array.from(new Set(granted)), userModels); + }, [isEditing, watchedModels, teamData, userModels]); const teamEditAccess = useMemo(() => parseTeamEditAccess(teamData?.team_info?.caller_edit_access), [teamData]); const canEditTeam = is_team_admin || is_proxy_admin || teamEditAccess.kind !== "none"; @@ -684,6 +694,9 @@ const TeamInfoView: React.FC = ({ const startEditing = () => { form.reset(teamFormValues()); setTeamModelMaxBudget((teamData?.team_info?.model_max_budget ?? {}) as ModelMaxBudget); + setTeamMemberModelMaxBudget( + (teamData?.team_info?.team_member_budget_table?.model_max_budget ?? {}) as ModelMaxBudget, + ); setTeamMemberSettingsOpen(false); setSearchToolSettingsOpen(false); setIsEditing(true); @@ -866,6 +879,7 @@ const TeamInfoView: React.FC = ({ allowed_models: values.allowed_models, temp_budget_increase: values.temp_budget_increase, temp_budget_expiry: values.temp_budget_expiry, + model_max_budget: modelMaxBudgetUpdate(values.model_max_budget ?? {}, selectedEditMember?.model_max_budget), }; toast.dismiss(); // Remove all existing toasts @@ -1223,6 +1237,14 @@ const TeamInfoView: React.FC = ({ updateData.model_max_budget = modelBudgets; } + const teamMemberModelBudgets = modelMaxBudgetUpdate( + teamMemberModelMaxBudget, + info.team_member_budget_table?.model_max_budget, + ); + if (teamMemberModelBudgets !== undefined) { + updateData.team_member_model_max_budget = teamMemberModelBudgets; + } + // Handle router_settings - read fresh values from DOM at save time. const currentRouterSettings = routerSettingsRef.current?.getValue(); if (currentRouterSettings?.router_settings) { @@ -1658,6 +1680,14 @@ const TeamInfoView: React.FC = ({ /> )} + = ({

Max Budget: {info.team_member_budget_table?.max_budget ?? "No Limit"}
Budget Duration: {info.team_member_budget_table?.budget_duration || "No Limit"}
+ {modelMaxBudgetToEntries( + info.team_member_budget_table?.model_max_budget as ModelMaxBudget | null | undefined, + ).map(({ model, budgetLimit, timePeriod }) => ( +
+ Default Per-Model Budget ({model}):{" "} + {budgetLimit !== null ? `$${formatNumberWithCommas(budgetLimit, 4)}` : "No Limit"} per {timePeriod} +
+ ))}
Key Duration: {info.metadata?.team_member_key_duration || "No Limit"}
TPM Limit: {info.team_member_budget_table?.tpm_limit ?? "No Limit"}
RPM Limit: {info.team_member_budget_table?.rpm_limit ?? "No Limit"}
@@ -2554,6 +2592,20 @@ const TeamInfoView: React.FC = ({ ), type: "utc-datetime" as const, }, + { + name: "model_max_budget", + label: ( + + Per-Model Budgets{" "} + + + + + ), + type: "model-max-budget" as const, + availableModels: availableModelBudgetModels, + premiumUser, + }, { name: "tpm_limit", label: ( diff --git a/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx b/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx index 8953a06796f..b8f292222ed 100644 --- a/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx @@ -432,10 +432,31 @@ describe("TeamMembersComponent", () => { allowed_models: [], temp_budget_increase: 0, temp_budget_expiry: "2030-01-02T03:04:00Z", + model_max_budget: null, }; expect(seedMemberBudgetFields({ user_id: "user1@test.com", role: "member" }, budget)).toStrictEqual(seeded); }); + it("seeds the team's default per-model budget when the member has no per-model override", () => { + const teamDefault = { "gpt-4o": { max_budget: 10, budget_duration: "30d" } }; + const ownRowWithoutModelCaps = { + ...createMockTeamData().team_memberships[0].litellm_budget_table, + model_max_budget: {}, + }; + const memberOverride = { "gpt-4o": { max_budget: 2.5, budget_duration: "7d" } }; + const ownRowWithModelCaps = { ...ownRowWithoutModelCaps, model_max_budget: memberOverride }; + + expect( + seedMemberBudgetFields({ user_id: "u", role: "user" }, ownRowWithoutModelCaps, teamDefault).model_max_budget, + ).toBe(teamDefault); + expect(seedMemberBudgetFields({ user_id: "u", role: "user" }, undefined, teamDefault).model_max_budget).toBe( + teamDefault, + ); + expect( + seedMemberBudgetFields({ user_id: "u", role: "user" }, ownRowWithModelCaps, teamDefault).model_max_budget, + ).toBe(memberOverride); + }); + it("seeds null temporary budget fields for a member without a budget row", () => { expect(seedMemberBudgetFields({ user_id: "user2@test.com", role: "admin" }, undefined)).toMatchObject({ max_budget_in_team: null, diff --git a/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx b/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx index 6778b40e93c..f50c47dd12f 100644 --- a/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx @@ -7,6 +7,7 @@ import { Button } from "@/components/ui/button"; import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { SimpleTooltip } from "@/components/ui/tooltip"; import MemberTable from "@/components/common_components/MemberTable"; +import type { StoredModelMaxBudget } from "@/components/key_team_helpers/modelMaxBudgetPayload"; import { Member } from "@/components/networking"; import { parseErrorMessage } from "@/components/shared/errorUtils"; import { DateCell, MoneyCell } from "@/components/shared/table_cells"; @@ -25,9 +26,13 @@ const BUDGET_SOURCE_LABELS: Record, stri const formatBudget = (value: number | null): string => value === null ? "Unlimited" : `$${formatNumberWithCommas(value, 2)}`; +const hasModelBudgets = (budget: StoredModelMaxBudget | null | undefined): budget is StoredModelMaxBudget => + budget != null && Object.keys(budget).length > 0; + export const seedMemberBudgetFields = ( record: Member, budget: TeamMembership["litellm_budget_table"] | undefined, + teamDefaultModelMaxBudget?: StoredModelMaxBudget | null, ): Member => ({ ...record, max_budget_in_team: budget?.max_budget ?? null, @@ -37,6 +42,9 @@ export const seedMemberBudgetFields = ( allowed_models: budget?.allowed_models || [], temp_budget_increase: budget?.temp_budget_increase ?? null, temp_budget_expiry: budget?.temp_budget_expiry ?? null, + model_max_budget: hasModelBudgets(budget?.model_max_budget) + ? budget.model_max_budget + : teamDefaultModelMaxBudget ?? null, }); interface TeamMemberTabProps { @@ -305,7 +313,13 @@ export default function TeamMemberTab({ canEdit={canEditTeam} onEdit={(record) => { const membership = teamData.team_memberships.find((tm) => tm.user_id === record.user_id); - setSelectedEditMember(seedMemberBudgetFields(record, membership?.litellm_budget_table)); + setSelectedEditMember( + seedMemberBudgetFields( + record, + membership?.litellm_budget_table, + teamData.team_info.team_member_budget_table?.model_max_budget, + ), + ); setIsEditMemberModalVisible(true); }} onDelete={handleMemberDelete} diff --git a/ui/litellm-dashboard/src/components/team/memberFormValues.ts b/ui/litellm-dashboard/src/components/team/memberFormValues.ts index 77cf2dd3e43..df80e2dd9ac 100644 --- a/ui/litellm-dashboard/src/components/team/memberFormValues.ts +++ b/ui/litellm-dashboard/src/components/team/memberFormValues.ts @@ -1,8 +1,21 @@ -export type MemberFieldValue = string | number | null | undefined | string[]; +import { + entriesToModelMaxBudget, + modelMaxBudgetToEntries, + type ModelMaxBudget, +} from "../key_team_helpers/ModelMaxBudgetEditor"; + +export type MemberFieldValue = string | number | null | undefined | string[] | ModelMaxBudget; export type MemberFormValues = Record; -export type MemberFieldType = "input" | "select" | "numerical" | "multi-select" | "budget-duration" | "utc-datetime"; +export type MemberFieldType = + | "input" + | "select" + | "numerical" + | "multi-select" + | "budget-duration" + | "utc-datetime" + | "model-max-budget"; export interface MemberAdditionalField { name: string; @@ -12,6 +25,8 @@ export interface MemberAdditionalField { step?: number; min?: number; placeholder?: string; + availableModels?: string[]; + premiumUser?: boolean; } export interface MemberFieldsConfig { @@ -33,6 +48,9 @@ export const TEMP_BUDGET_PAIR_MESSAGE = "Set both a temporary budget increase an const isUnset = (value: MemberFieldValue): boolean => value === null || value === undefined || value === ""; +export const isModelMaxBudget = (value: MemberFieldValue): value is ModelMaxBudget => + typeof value === "object" && value !== null && !Array.isArray(value); + export const tempBudgetPairError = (values: MemberFormValues): "temp_budget_increase" | "temp_budget_expiry" | null => { const increaseUnset = isUnset(values.temp_budget_increase); const expiryUnset = isUnset(values.temp_budget_expiry); @@ -66,6 +84,9 @@ export const buildMemberFormValues = ( allowed_models: initialData.allowed_models || [], temp_budget_increase: initialData.temp_budget_increase ?? null, temp_budget_expiry: initialData.temp_budget_expiry || null, + model_max_budget: isModelMaxBudget(initialData.model_max_budget) + ? entriesToModelMaxBudget(modelMaxBudgetToEntries(initialData.model_max_budget)) + : {}, }; return pickFieldNames(config, seeded); @@ -78,6 +99,8 @@ const emptyValueForType = (type: MemberFieldType | undefined): MemberFieldValue switch (type) { case "multi-select": return []; + case "model-max-budget": + return {}; case "numerical": case "budget-duration": case "utc-datetime": diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index f36456628e5..731a92d8b98 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -16882,6 +16882,7 @@ export interface paths { * - object_permission: Optional[LiteLLM_ObjectPermissionBase] - team-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"], "agents": ["agent_1", "agent_2"], "agent_access_groups": ["dev_group"]}. IF null or {} then no object permission. * - team_member_budget: Optional[float] - The maximum budget allocated to an individual team member. * - team_member_budget_duration: Optional[str] - The duration of the budget for the team member. Doc [here](https://docs.litellm.ai/docs/proxy/team_budgets) + * - team_member_model_max_budget: Optional[dict] - Per-model budgets shared by team members without an override. * - team_member_rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for individual team members. * - team_member_tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for individual team members. * - team_member_key_duration: Optional[str] - The duration for a team member's key. e.g. "1d", "1w", "1mo" @@ -17111,6 +17112,7 @@ export interface paths { * - object_permission: Optional[LiteLLM_ObjectPermissionBase] - team-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"], "agents": ["agent_1", "agent_2"], "agent_access_groups": ["dev_group"]}. IF null or {} then no object permission. * - team_member_budget: Optional[float] - The maximum budget allocated to an individual team member. * - team_member_budget_duration: Optional[str] - The duration of the budget for the team member. Doc [here](https://docs.litellm.ai/docs/proxy/team_budgets) + * - team_member_model_max_budget: Optional[dict] - Per-model budgets shared by team members without an override. * - team_member_rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for individual team members. * - team_member_tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for individual team members. * - team_member_key_duration: Optional[str] - The duration for a team member's key. e.g. "1d", "1w", "1mo" @@ -38499,6 +38501,10 @@ export interface components { team_member_budget_duration?: string | null; /** Team Member Key Duration */ team_member_key_duration?: string | null; + /** Team Member Model Max Budget */ + team_member_model_max_budget?: { + [key: string]: components["schemas"]["BudgetConfig"]; + } | null; /** Team Member Permissions */ team_member_permissions?: string[] | null; /** Team Member Rpm Limit */ @@ -39584,6 +39590,10 @@ export interface components { team_member_budget_duration?: string | null; /** Team Member Key Duration */ team_member_key_duration?: string | null; + /** Team Member Model Max Budget */ + team_member_model_max_budget?: { + [key: string]: components["schemas"]["BudgetConfig"]; + } | null; /** Team Member Rpm Limit */ team_member_rpm_limit?: number | null; /** Team Member Tpm Limit */ @@ -45580,6 +45590,13 @@ export interface components { budget_duration?: string | null; /** Max Budget In Team */ max_budget_in_team?: number | null; + /** + * Model Max Budget + * @description Per-model spend caps for this team member, each with its own budget_duration. Overrides the team's default per-model member budget. Pass an empty dict to fall back to the team default. + */ + model_max_budget?: { + [key: string]: components["schemas"]["BudgetConfig"]; + } | null; /** Role */ role?: ("admin" | "user") | null; /** @@ -45617,6 +45634,10 @@ export interface components { budget_duration?: string | null; /** Max Budget In Team */ max_budget_in_team?: number | null; + /** Model Max Budget */ + model_max_budget?: { + [key: string]: components["schemas"]["BudgetConfig"]; + } | null; /** Rpm Limit */ rpm_limit?: number | null; /** Team Id */ @@ -47559,6 +47580,10 @@ export interface components { team_member_budget_duration?: string | null; /** Team Member Key Duration */ team_member_key_duration?: string | null; + /** Team Member Model Max Budget */ + team_member_model_max_budget?: { + [key: string]: components["schemas"]["BudgetConfig"]; + } | null; /** Team Member Rpm Limit */ team_member_rpm_limit?: number | null; /** Team Member Tpm Limit */ From 12c36b054b2247b47eb9ccbc1aa7a3ae0cefe249 Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 13:33:58 -0700 Subject: [PATCH 07/40] refactor(constants): move AUTH_OBJECTS_TARGET into litellm.constants (#45779) * refactor(constants): move AUTH_OBJECTS_TARGET into litellm.constants Part of LIT-9423 * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/integrations/prometheus.py | 2 +- litellm/proxy/common_utils/user_api_key_cache.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 8252e923ae5..a339fb76976 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1969,6 +1969,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ ] SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"] DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)) +AUTH_OBJECTS_TARGET: Final = "auth_objects" DEFAULT_ACCESS_GROUP_CACHE_TTL: Final = int(os.getenv("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600)) SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = 600 SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL: Final = 30 diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 8f4d1df2662..797c443e712 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -21,6 +21,7 @@ import litellm from litellm._internal_context import with_service_target from litellm._logging import print_verbose, verbose_logger from litellm.constants import ( + AUTH_OBJECTS_TARGET, PROMETHEUS_OVERFLOW_SERIES_LABEL_VALUE, PROXY_LLM_PROVIDER_FALLBACK, PROXY_REJECTED_BEFORE_ROUTING_KEY, @@ -53,7 +54,6 @@ from litellm.models.team import LiteLLM_TeamTable from litellm.models.user import LiteLLM_UserTable from litellm.models.verification_token import LiteLLM_DeletedVerificationToken from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET from litellm.repositories.base_repository import BaseRepository from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.organization_repository import OrganizationRepository diff --git a/litellm/proxy/common_utils/user_api_key_cache.py b/litellm/proxy/common_utils/user_api_key_cache.py index 8c830816207..e64fd755ef9 100644 --- a/litellm/proxy/common_utils/user_api_key_cache.py +++ b/litellm/proxy/common_utils/user_api_key_cache.py @@ -21,7 +21,7 @@ if TYPE_CHECKING: T = TypeVar("T", bound=BaseModel) -AUTH_OBJECTS_TARGET: Final = "auth_objects" +from litellm.constants import AUTH_OBJECTS_TARGET as AUTH_OBJECTS_TARGET # noqa: E402, PLC0414 # public re-export _HASHED_TOKEN_CACHE_KEY: Final = re.compile(r"[0-9a-f]{64}") From 2374b0c398865e332f6b3b77f31a09ef707d4ed5 Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 13:34:39 -0700 Subject: [PATCH 08/40] refactor(types): annotate key request containers and drop_blank_team_id (#45809) * refactor(types): annotate key request containers and drop_blank_team_id Part of LIT-9423 * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7ac8b1f8538..9ee20c22cef 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1243,8 +1243,8 @@ class KeyRequestBase(GenerateRequestBase): enable_prompt_caching: bool | None = None throttle_on_budget_exceeded: bool | None = None enforced_params: list[str] | None = None - allowed_routes: list | None = [] - allowed_passthrough_routes: list | None = None + allowed_routes: list[Any] | None = [] + allowed_passthrough_routes: list[object] | None = None denied_passthrough_routes: list[str] | None = None allowed_vector_store_indexes: list[AllowedVectorStoreIndexItem] | None = None rpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = ( @@ -1333,13 +1333,17 @@ class GenerateKeyResponse(KeyRequestBase): return values +def _is_mapping(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: raw before-validator payload + return isinstance(value, Mapping) + + class UpdateKeyRequest(KeyRequestBase): # Note: the defaults of all Params here MUST BE NONE # else they will get overwritten duration: str | None = None spend: float | None = None soft_budget: float | None = None - metadata: dict | None = None + metadata: dict[Any, Any] | None = None temp_budget_increase: float | None = None temp_budget_expiry: datetime | None = None auto_rotate: bool | None = None @@ -1354,7 +1358,7 @@ class UpdateKeyRequest(KeyRequestBase): @model_validator(mode="before") @classmethod def drop_blank_team_id(cls, values: object) -> object: - if isinstance(values, Mapping) and values.get("team_id") == "": + if _is_mapping(values) and values.get("team_id") == "": return MappingProxyType({k: v for k, v in values.items() if k != "team_id"}) return values @@ -1385,7 +1389,7 @@ class RegenerateKeyRequest(GenerateKeyRequest): new_key: str | None = None duration: str | None = None spend: float | None = None - metadata: dict | None = None + metadata: dict[Any, Any] | None = None new_master_key: str | None = None grace_period: str | None = None # Duration to keep old key valid (e.g. "24h", "2d"); None = immediate revoke From 981710fab81c5551c09717b66da57ed41504885a Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 13:35:07 -0700 Subject: [PATCH 09/40] refactor(types): move GenerateRequestBase and LiteLLM_ObjectPermissionBase into litellm.types.proxy.management_endpoints.request_base (#45802) * refactor(types): move LitellmUserRoles and KeyManagementRoutes into litellm.types.proxy.auth.user_roles Part of LIT-9423 * refactor(types): annotate GenerateRequestBase containers and check_max_budget Part of LIT-9423 * refactor(types): move GenerateRequestBase and LiteLLM_ObjectPermissionBase into litellm.types.proxy.management_endpoints.request_base Part of LIT-9423 * fix(types): restore main's _types.py content alongside the request_base move Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * style(types): merge request_base re-exports into one import block Part of LIT-9423 --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 67 ++----------------- .../management_endpoints/request_base.py | 67 +++++++++++++++++++ 2 files changed, 73 insertions(+), 61 deletions(-) create mode 100644 litellm/types/proxy/management_endpoints/request_base.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9ee20c22cef..0715a4cae97 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1159,71 +1159,16 @@ class ModelParams(LiteLLMPydanticObjectBase): return values -class LiteLLM_ObjectPermissionBase(LiteLLMPydanticObjectBase): - mcp_servers: list[str] | None = None - mcp_access_groups: list[str] | None = None - mcp_tool_permissions: dict[str, list[str]] | None = None - mcp_toolsets: list[str] | None = None - blocked_tools: list[str] | None = None - vector_stores: list[str] | None = None - agents: list[str] | None = None - agent_access_groups: list[str] | None = None - models: list[str] | None = None - search_tools: list[str] | None = None - mcp_tool_search_enabled: bool | None = None - skills: list[str] | None = None - - from litellm.models.team import BudgetLimitEntry as BudgetLimitEntry # noqa: E402 from litellm.types.object_permission import ( # noqa: E402 ObjectPermissionDict as ObjectPermissionDict, ) - - -class GenerateRequestBase(LiteLLMPydanticObjectBase): - """ - Overlapping schema between key and user generate/update requests - """ - - key_alias: str | None = None - duration: str | None = None - models: list[Any] | None = [] - spend: float | None = 0 - max_budget: float | None = None - user_id: str | None = None - team_id: str | None = None - agent_id: str | None = None - max_parallel_requests: int | None = None - metadata: dict[Any, Any] | None = {} - tpm_limit: int | None = None - rpm_limit: int | None = None - - budget_duration: str | None = None - budget_limits: list[BudgetLimitEntry] | None = None # multiple concurrent budget windows - allowed_cache_controls: list[object] | None = [] - config: dict[object, object] | None = {} - permissions: dict[object, object] | None = {} - model_max_budget: dict[object, object] | None = {} # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {} - budget_fallbacks: dict[str, list[str]] | None = None - - model_config = ConfigDict(protected_namespaces=()) - model_rpm_limit: dict[object, object] | None = None - model_tpm_limit: dict[object, object] | None = None - mcp_rpm_limit: dict[str, int] | None = None - tag_rpm_limit: dict[str, int] | None = None - guardrails: list[str] | None = None - policies: list[str] | None = None - prompts: list[str] | None = None - blocked: bool | None = None - aliases: dict[object, object] | None = {} - object_permission: LiteLLM_ObjectPermissionBase | None = None - - @field_validator("max_budget", mode="before") - @classmethod - def check_max_budget(cls, v: object) -> object: - if v == "": - return None - return v +from litellm.types.proxy.management_endpoints.request_base import ( # noqa: E402 # re-export after the definitions above + GenerateRequestBase as GenerateRequestBase, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.request_base import ( # noqa: E402 # re-export after the definitions above + LiteLLM_ObjectPermissionBase as LiteLLM_ObjectPermissionBase, # noqa: PLC0414 # public re-export +) class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase): diff --git a/litellm/types/proxy/management_endpoints/request_base.py b/litellm/types/proxy/management_endpoints/request_base.py new file mode 100644 index 00000000000..6b4a76e0132 --- /dev/null +++ b/litellm/types/proxy/management_endpoints/request_base.py @@ -0,0 +1,67 @@ +from typing import Any # noqa: TID251 # metadata and models stay Any so existing callers keep type-checking + +from pydantic import ConfigDict, field_validator + +from litellm.models.team import BudgetLimitEntry +from litellm.types.utils import LiteLLMPydanticObjectBase + + +class LiteLLM_ObjectPermissionBase(LiteLLMPydanticObjectBase): + mcp_servers: list[str] | None = None + mcp_access_groups: list[str] | None = None + mcp_tool_permissions: dict[str, list[str]] | None = None + mcp_toolsets: list[str] | None = None + blocked_tools: list[str] | None = None + vector_stores: list[str] | None = None + agents: list[str] | None = None + agent_access_groups: list[str] | None = None + models: list[str] | None = None + search_tools: list[str] | None = None + mcp_tool_search_enabled: bool | None = None + skills: list[str] | None = None + + +class GenerateRequestBase(LiteLLMPydanticObjectBase): + """ + Overlapping schema between key and user generate/update requests + """ + + key_alias: str | None = None + duration: str | None = None + models: list[Any] | None = [] + spend: float | None = 0 + max_budget: float | None = None + user_id: str | None = None + team_id: str | None = None + agent_id: str | None = None + max_parallel_requests: int | None = None + metadata: dict[Any, Any] | None = {} + tpm_limit: int | None = None + rpm_limit: int | None = None + + budget_duration: str | None = None + budget_limits: list[BudgetLimitEntry] | None = None # multiple concurrent budget windows + allowed_cache_controls: list[object] | None = [] + config: dict[object, object] | None = {} + permissions: dict[object, object] | None = {} + model_max_budget: dict[object, object] | None = {} # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {} + budget_fallbacks: dict[str, list[str]] | None = None + + model_config = ConfigDict(protected_namespaces=()) + model_rpm_limit: dict[object, object] | None = None + model_tpm_limit: dict[object, object] | None = None + mcp_rpm_limit: dict[str, int] | None = None + tag_rpm_limit: dict[str, int] | None = None + guardrails: list[str] | None = None + policies: list[str] | None = None + prompts: list[str] | None = None + blocked: bool | None = None + aliases: dict[object, object] | None = {} + object_permission: LiteLLM_ObjectPermissionBase | None = None + + @field_validator("max_budget", mode="before") + @classmethod + def check_max_budget(cls, v: object) -> object: + if v == "": + return None + return v From f58c86f1d711ca70f7202ee493f1bd8c0d5fe5ad Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 10 Oct 2026 13:35:11 -0700 Subject: [PATCH 10/40] perf(ci): use native timing splits with bounded integration concurrency (#45837) * perf(ci): use native timing splits with bounded integration concurrency * fix(ci): make integration parallelism opt in --- .circleci/config.yml | 123 +++++------ .circleci/scripts/run_integration.sh | 21 +- tests/integration/README.md | 14 +- tests/integration/_support/ordering.py | 83 -------- tests/integration/_support/sharding.py | 17 -- tests/integration/conftest.py | 54 ++--- .../database/test_harness_lifecycle.py | 194 +++++++++++++----- tests/integration/run.py | 12 +- .../unit/integration_support/test_ordering.py | 115 ----------- .../unit/integration_support/test_sharding.py | 58 +++--- 10 files changed, 283 insertions(+), 408 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b22c2b4b7a9..bfca8310c0e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2747,33 +2747,6 @@ jobs: - store_test_results: path: test-results/provider-replay-harness - integration_timing_snapshot: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: medium - working_directory: ~/project - steps: - - checkout - - run: - name: Prepare optional integration timing snapshots - command: mkdir -p /tmp/integration-timings/{extensions,providers,mcp} - - restore_cache: - keys: - - integration-timings-v1-extensions-{{ .Branch }}-{{ .Revision }} - - integration-timings-v1-extensions-{{ .Branch }}- - - restore_cache: - keys: - - integration-timings-v1-providers-{{ .Branch }}-{{ .Revision }} - - integration-timings-v1-providers-{{ .Branch }}- - - restore_cache: - keys: - - integration-timings-v1-mcp-{{ .Branch }}-{{ .Revision }} - - integration-timings-v1-mcp-{{ .Branch }}- - - persist_to_workspace: - root: /tmp - paths: - - integration-timings - integration_contracts: parameters: suite: @@ -2796,15 +2769,6 @@ jobs: parallelism: << parameters.parallelism >> steps: - setup_litellm_test_deps - - when: - condition: - or: - - equal: [extensions, << parameters.suite >>] - - equal: [providers, << parameters.suite >>] - - equal: [mcp, << parameters.suite >>] - steps: - - attach_workspace: - at: /tmp - when: condition: equal: [browser, << parameters.suite >>] @@ -2850,24 +2814,26 @@ jobs: test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)" - when: condition: - or: - - equal: [extensions, << parameters.suite >>] - - equal: [providers, << parameters.suite >>] - - equal: [mcp, << parameters.suite >>] + not: + equal: [1, << parameters.parallelism >>] steps: - run: name: Prepare this shard's execution record command: | - mkdir -p "/tmp/integration-shards/<< parameters.suite >>/$CIRCLE_NODE_INDEX" - cp test-results/integration-<< parameters.suite >>/execution.json "/tmp/integration-shards/<< parameters.suite >>/$CIRCLE_NODE_INDEX/execution.json" + results="test-results/integration-<< parameters.suite >>" + if [ "<< parameters.mode >>" = replica ]; then + results="$results-replica" + fi + mkdir -p "/tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >>/$CIRCLE_NODE_INDEX" + cp "$results/execution.json" "/tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >>/$CIRCLE_NODE_INDEX/execution.json" if [ "<< parameters.suite >>" = mcp ]; then - mkdir -p "/tmp/integration-shards/mcp/$CIRCLE_NODE_INDEX/coverage" - cp test-results/integration-mcp/coverage/data "/tmp/integration-shards/mcp/$CIRCLE_NODE_INDEX/coverage/data.$CIRCLE_NODE_INDEX" + mkdir -p "/tmp/integration-shards/mcp-standard/$CIRCLE_NODE_INDEX/coverage" + cp test-results/integration-mcp/coverage/data "/tmp/integration-shards/mcp-standard/$CIRCLE_NODE_INDEX/coverage/data.$CIRCLE_NODE_INDEX" fi - persist_to_workspace: root: /tmp/integration-shards paths: - - << parameters.suite >> + - << parameters.suite >>-<< parameters.mode >> - store_test_results: path: test-results - store_artifacts: @@ -2877,7 +2843,11 @@ jobs: parameters: suite: type: enum - enum: [extensions, providers, mcp] + enum: [management, accounting, database, extensions, providers, mcp] + mode: + type: enum + enum: [standard, replica] + default: standard shards: type: integer machine: @@ -2891,12 +2861,8 @@ jobs: - run: name: Verify every << parameters.suite >> case ran exactly once command: | - .venv/bin/python -m tests.integration._support.sharding /tmp/integration-shards/<< parameters.suite >> \ - --shards << parameters.shards >> --timings-output /tmp/integration-timings/<< parameters.suite >>/timings.json - - save_cache: - key: integration-timings-v1-<< parameters.suite >>-{{ .Branch }}-{{ .Revision }} - paths: - - /tmp/integration-timings/<< parameters.suite >> + .venv/bin/python -m tests.integration._support.sharding /tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >> \ + --shards << parameters.shards >> - when: condition: equal: [mcp, << parameters.suite >>] @@ -2906,7 +2872,7 @@ jobs: command: | mkdir -p test-results/integration-mcp/coverage export COVERAGE_FILE="$PWD/test-results/integration-mcp/coverage/data" - .venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml /tmp/integration-shards/mcp/*/coverage + .venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml /tmp/integration-shards/mcp-standard/*/coverage .venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > test-results/integration-mcp/coverage/coverage.txt .venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d test-results/integration-mcp/coverage/html - store_artifacts: @@ -3205,48 +3171,59 @@ workflows: - not: equal: ["", << pipeline.parameters.routing_parity_base >>] jobs: - - integration_timing_snapshot: - name: integration-timing-snapshot - integration_contracts: name: integration-<< matrix.suite >> matrix: parameters: suite: [sdk, cost, browser] + parallelism: 1 - integration_contracts: name: integration-security suite: security + parallelism: 1 resource_class: xlarge - integration_contracts: name: integration-management suite: management - parallelism: 8 + parallelism: 4 + - integration_shard_coverage: + name: integration-management-coverage + suite: management + shards: 4 + requires: [integration-management] - integration_contracts: name: integration-accounting suite: accounting - parallelism: 8 + parallelism: 4 + - integration_shard_coverage: + name: integration-accounting-coverage + suite: accounting + shards: 4 + requires: [integration-accounting] - integration_contracts: name: integration-providers suite: providers - parallelism: 16 - requires: - - integration-timing-snapshot + parallelism: 8 - integration_shard_coverage: name: integration-providers-coverage suite: providers - shards: 16 + shards: 8 requires: - integration-providers - integration_contracts: name: integration-database suite: database parallelism: 2 + - integration_shard_coverage: + name: integration-database-coverage + suite: database + shards: 2 + requires: [integration-database] - integration_contracts: name: integration-mcp suite: mcp resource_class: xlarge parallelism: 2 - requires: - - integration-timing-snapshot - integration_shard_coverage: name: integration-mcp-coverage suite: mcp @@ -3256,25 +3233,35 @@ workflows: - integration_contracts: name: integration-extensions suite: extensions - parallelism: 32 - requires: - - integration-timing-snapshot + parallelism: 8 - integration_shard_coverage: name: integration-extensions-coverage suite: extensions - shards: 32 + shards: 8 requires: - integration-extensions - integration_contracts: name: integration-management-replica suite: management mode: replica - parallelism: 8 + parallelism: 4 + - integration_shard_coverage: + name: integration-management-replica-coverage + suite: management + mode: replica + shards: 4 + requires: [integration-management-replica] - integration_contracts: name: integration-database-replica suite: database mode: replica parallelism: 2 + - integration_shard_coverage: + name: integration-database-replica-coverage + suite: database + mode: replica + shards: 2 + requires: [integration-database-replica] - postgres_suite: name: proxy-behavior test_path: tests/proxy_behavior diff --git a/.circleci/scripts/run_integration.sh b/.circleci/scripts/run_integration.sh index 4bdef5eb23b..28d1fba68c7 100644 --- a/.circleci/scripts/run_integration.sh +++ b/.circleci/scripts/run_integration.sh @@ -216,18 +216,11 @@ if [ "$suite" = browser ]; then exit 0 fi -node_files=() -case_shards=1 -case_shard_index=0 -if [ "$suite" = extensions ] || [ "$suite" = providers ] || [ "$suite" = mcp ]; then - case_shards="${CIRCLE_NODE_TOTAL:-1}" - case_shard_index="${CIRCLE_NODE_INDEX:-0}" -elif [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then - split="$(.venv/bin/python tests/integration/run.py "$suite" --list \ - | circleci tests split --split-by=timings --timings-type=filename)" - read -r -a node_files <<< "$(printf '%s' "$split" | tr '\n' ' ')" - test "${#node_files[@]}" -gt 0 - printf '%s\n' "${node_files[@]}" > "$results/node-files.txt" +shard_arguments=() +if [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then + .venv/bin/python tests/integration/run.py "$suite" --list \ + | circleci tests split --split-by=timings --timings-type=filename > "$results/node-files.txt" + shard_arguments=(--shard-files "$results/node-files.txt") fi env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \ @@ -240,13 +233,13 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \ LITELLM_LICENSE="${LITELLM_LICENSE:-}" \ INTEGRATION_SEED="$INTEGRATION_SEED" \ INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \ - INTEGRATION_CASE_SHARDS="$case_shards" INTEGRATION_CASE_SHARD_INDEX="$case_shard_index" \ + INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \ LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \ INTEGRATION_PROXY_DATABASE_URL="$INTEGRATION_PROXY_DATABASE_URL" \ INTEGRATION_PROXY_READ_REPLICA_URL="$INTEGRATION_PROXY_READ_REPLICA_URL" \ INTEGRATION_ROUTING="$INTEGRATION_ROUTING" \ .venv/bin/python tests/integration/run.py "$suite" --results "$results" \ - --timings "/tmp/integration-timings/$suite/timings.json" "${node_files[@]}" + "${shard_arguments[@]}" if [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then for covered_pid in "$proxy_pid" "$peer_pid"; do diff --git a/tests/integration/README.md b/tests/integration/README.md index 8734e2e6237..3bbfc8e386e 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -10,7 +10,15 @@ Use `tests/integration/run.py management`, `accounting`, `database`, `providers` Management also requires `INTEGRATION_PEER_URL`, `REDIS_HOST` and `REDIS_PORT`. CircleCI starts two directly addressed proxy processes sharing only that job's stores. The test-only CLI wrapper supplies enterprise route entitlement, following the existing behavior suite's convention. It does not qualify license validation; run it with one worker and no reload -The generated lifecycle models use 20 examples, eight steps, generation and shrinking, with isolated resources per example. HTTP operation caps include generation and shrinking and exempt cleanup. Local qualification defaults to seed 4106601 and canonical order; CircleCI derives exploration and ordering seeds from the checked-out revision and workflow ID. The ordering seed shuffles files, scoped fixture configurations and cases within each configuration. Cases sharing a module or class fixture parameter stay together, so shuffling does not repeatedly rebuild the same configuration. Case partitions also group tests using the same unparametrized module or class fixtures. These groups split when they exceed a node's estimated time budget, limiting repeated setup without placing an oversized group on one node. A successful shard census saves every case's setup, call and teardown duration. The next CircleCI run restores that shared snapshot and balances measured cost. Missing or invalid timing data falls back to equal case weights. New cases receive the median known cost, and stale case identities cannot select or omit tests. Use `--timings ` to reproduce a measured partition Use `--seed` and `--order-seed` to reproduce a run. Actual installed Hypothesis version, settings, seeds and collected order are written beside the execution manifest +The generated lifecycle models use 20 examples, eight steps, generation and shrinking, with isolated resources per example. HTTP operation caps include generation and shrinking and exempt cleanup. Local qualification defaults to seed 4106601 and canonical order; CircleCI derives exploration and ordering seeds from the checked-out revision and workflow ID. The ordering seed shuffles files, scoped fixture configurations and cases within each configuration. Cases sharing a module or class fixture parameter stay together, so shuffling does not repeatedly rebuild the same configuration + +CircleCI uses its built-in `circleci tests split --split-by=timings --timings-type=filename` to assign test files using the uploaded JUnit timings. Each file stays on one node, preserving its module and parametrized fixtures. Jobs default to one node. Parallelism should use the smallest node count that keeps the complete job, including setup and cleanup, within twenty minutes. A group that already finishes within that target on one node stays on one node. Management, accounting and management replica currently use four nodes. Providers and extensions use eight nodes; database and MCP use two, and fast suites use one. No job is configured above eight nodes. A single slow file can still determine a suite's runtime; split such a file along independent fixture boundaries rather than adding more nodes + +Every node collects the same candidate files before applying CircleCI's assigned file list. Its execution record retains the candidate inventory and the selected, passed and skipped cases. An empty assigned file list produces an empty shard, not a full-suite fallback. Every parallel group has a dependent census job that rejects an empty candidate inventory, missing shards, omitted cases and duplicate execution. Standard and replica runs keep separate workspace records + +Future change-based selection belongs before timing splitting. Pass the selected group files or node IDs to `run.py --list` to list their files, then give those filenames to CircleCI. Run the same candidate selection on every node with `--shard-files --shards --shard-index `. The census compares execution with the candidate inventory, while full-suite runs remain the default. CircleCI determines scheduling from timing history; timing data never selects or omits candidates. Use `--seed` and `--order-seed` to reproduce test generation and order. The installed Hypothesis version, settings, seeds and collected order are recorded beside the execution manifest + +A future selector must explicitly skip a group when no candidates apply; omitting positional candidates means the full group. Keep full runs on the default branch and scheduled workflows, and select the full affected groups when shared fixtures, CI configuration or an unknown dependency changes Reuse the existing canned provider handlers through `_support/upstream.py`. It rejects internal request fields and exposes actual received requests for independent assertions. Register every created resource for cleanup immediately, keep expected values independent of production calculations, and assert readback plus the runtime effect of a change @@ -34,7 +42,7 @@ Accounting cases compare persisted input and output cost components against lite Provider contracts exercise actual TCP requests with synthetic credentials and local protocol peers. The S3 verifier uses independently implemented equations, a published known-answer vector, a fixed signing clock and deliberately invalid signed requests. Bedrock cases clear ambient AWS credential sources and check the literal model path, loaded role references, STS requests and bearer-only behavior -CircleCI runs providers on 16 parallel nodes, partitioning cases while keeping scoped fixture configurations together. Each node owns its services. The dependent providers coverage job verifies all 16 execution records against the full case inventory and rejects missing, duplicate, failed or incomplete execution. Use `--shards 16 --shard-index ` to reproduce a node locally +CircleCI runs providers on eight isolated nodes using its file timing splitter. Each node owns its services. The dependent providers coverage job verifies all eight execution records against the candidate case inventory and rejects missing, duplicate, failed or incomplete execution. The node's assigned filenames are saved in `node-files.txt` beside its execution record Streaming checks send real HTTP transfer chunks, including one-byte partitions, fragmented tools, incomplete transfers and a cancellation barrier. They assert meaningful text, tool arguments, final usage and persisted cost. The Redis recovery case owns a separate database and Redis process, uses the supported one-second circuit-breaker recovery setting, waits for the real subscriber and verifies response data in Redis after restart. CircleCI reuses its existing Redis image for that extra process; it never pulls an image during tests @@ -42,7 +50,7 @@ The `messages_endpoint/` directory holds `/v1/messages` endpoint contracts: nati The sdk shard exercises the SDK's own HTTP clients against local protocol peers with no gateway in the path, so a case here fails only when the client library or its wire behavior changes. The HTTP/2 case runs a hypercorn TLS peer offering h2 and http/1.1 over ALPN, drives the sync and async httpx handlers at it with `LITELLM_HTTP2` off and on, and asserts the version both the client and the peer observed on the wire. Put a test here only when it needs no proxy or database. CircleCI starts a local Redis for this shard like the others, so SDK-side caching cases that need a real Redis server belong here too; a case that reaches the gateway belongs in one of the other shards -The extensions shard uses the built-in generic callback and guardrail transports. It checks callback correlation and credential exclusion, guardrail rewriting and denial, retained OpenAI consumers and A2A wire versions. CircleCI runs it on 32 parallel large nodes. Each node starts its own database, Redis, upstream and proxy and runs its share of cases serially. The runner assigns cases deterministically by estimated duration, keeping scoped fixture configurations together. Each node records the full collection inventory and its selected, passed and skipped nodes. The dependent coverage job rejects missing shards, differing inventories, omitted or duplicate cases and incomplete execution. Use `--shards 32 --shard-index ` to reproduce a node locally. More parallel nodes increase setup overhead and concurrent machine use. Tests keep the isolation of a serial run; they still must not assume a particular set of sibling files. `run.py --list` prints a group's files and `run.py ...` runs a subset of them +The extensions shard uses the built-in generic callback and guardrail transports. It checks callback correlation and credential exclusion, guardrail rewriting and denial, retained OpenAI consumers and A2A wire versions. CircleCI runs it on eight parallel large nodes using its file timing splitter. Each node starts its own database, Redis, upstream and proxy and runs its assigned files serially. The dependent coverage job verifies all eight execution records against the candidate case inventory. Tests must not assume a particular set of sibling files The security shard uses four pytest workers on an xlarge machine, each with a private database and Redis. Route sweeps use a new HTTP client for every call and share only the TLS verification context, so authentication and response cookies cannot cross callers. The canary rig supplies owned OpenAI and Anthropic endpoints and synthetic credentials and rejects attempted external egress diff --git a/tests/integration/_support/ordering.py b/tests/integration/_support/ordering.py index 7dbc85396ff..fc10c372a93 100644 --- a/tests/integration/_support/ordering.py +++ b/tests/integration/_support/ordering.py @@ -2,39 +2,12 @@ from __future__ import annotations from collections.abc import Iterable, Mapping from dataclasses import dataclass -from functools import reduce from itertools import chain, groupby -from math import ceil, isfinite -from statistics import median from types import MappingProxyType from typing import Final import pytest from _pytest.python import get_direct_param_fixture_func -from pydantic import BaseModel, ConfigDict, Field, FiniteFloat, TypeAdapter, ValidationError - -EMPTY_DURATIONS: Final[Mapping[str, float]] = MappingProxyType({}) - - -class CaseTiming(BaseModel): - model_config = ConfigDict(frozen=True, strict=True) - - nodeid: str = Field(min_length=1) - seconds: FiniteFloat = Field(ge=0) - - -TIMINGS: Final = TypeAdapter(tuple[CaseTiming, ...]) - - -def parse_timings(content: str) -> Mapping[str, float]: - try: - values: Final = TIMINGS.validate_json(content) - except ValidationError: - return EMPTY_DURATIONS - durations: Final = {value.nodeid: max(value.seconds, 0.001) for value in values} - if len(durations) != len(values) or not isfinite(sum(durations.values())): - return EMPTY_DURATIONS - return MappingProxyType(durations) @dataclass(frozen=True, slots=True) @@ -120,59 +93,3 @@ def _file_groups(cases: Iterable[CollectedCase]) -> tuple[tuple[str, str], ...]: def fixture_groups(cases: tuple[CollectedCase, ...]) -> Mapping[str, str]: files: Final = groupby(sorted(cases, key=_file), key=_file) return MappingProxyType(dict(chain.from_iterable(_file_groups(members) for _, members in files))) - - -def _allocate( - shards: tuple[tuple[str, ...], ...], group: tuple[str, ...], durations: Mapping[str, float] -) -> tuple[tuple[str, ...], ...]: - index: Final = min( - range(len(shards)), - key=lambda candidate: (sum(durations[nodeid] for nodeid in shards[candidate]), len(shards[candidate])), - ) - return tuple((*shard, *group) if candidate == index else shard for candidate, shard in enumerate(shards)) - - -def _add_case( - groups: tuple[tuple[str, ...], ...], case: CollectedCase, maximum: float, durations: Mapping[str, float] -) -> tuple[tuple[str, ...], ...]: - if not groups: - return ((case.nodeid,),) - if sum(durations[nodeid] for nodeid in groups[-1]) + durations[case.nodeid] > maximum: - return (*groups, (case.nodeid,)) - return (*groups[:-1], (*groups[-1], case.nodeid)) - - -def _partition_group( - cases: Iterable[CollectedCase], maximum: float, durations: Mapping[str, float] -) -> tuple[tuple[str, ...], ...]: - members: Final = tuple(sorted(cases, key=lambda case: case.nodeid)) - nodeids: Final = tuple(case.nodeid for case in members) - if members[0].parameters: - return (nodeids,) - return reduce(lambda groups, case: _add_case(groups, case, maximum, durations), members, ()) - - -def case_shards( - cases: tuple[CollectedCase, ...], count: int, durations: Mapping[str, float] = EMPTY_DURATIONS -) -> tuple[frozenset[str], ...]: - if count < 1: - raise ValueError("Integration shard count must be positive") - groups: Final = fixture_groups(cases) - known: Final = tuple(durations[case.nodeid] for case in cases if case.nodeid in durations) - default: Final = max(float(median(known)), 0.001) if known else 1.0 - weights: Final = MappingProxyType({case.nodeid: max(durations.get(case.nodeid, default), 0.001) for case in cases}) - - def key(case: CollectedCase) -> str: - return groups[case.nodeid] or case.nodeid - - maximum: Final = sum(weights.values()) / count if known else float(max(1, ceil(len(cases) / count))) - members: Final = tuple( - chain.from_iterable( - _partition_group(group, maximum, weights) for _, group in groupby(sorted(cases, key=key), key=key) - ) - ) - ordered: Final = tuple(sorted(members, key=lambda group: (-sum(weights[nodeid] for nodeid in group), group[0]))) - allocation: Final = reduce( - lambda shards, group: _allocate(shards, group, weights), ordered, tuple(() for _ in range(count)) - ) - return tuple(frozenset(shard) for shard in allocation) diff --git a/tests/integration/_support/sharding.py b/tests/integration/_support/sharding.py index 5267985ad17..661b9626e17 100644 --- a/tests/integration/_support/sharding.py +++ b/tests/integration/_support/sharding.py @@ -9,15 +9,12 @@ from typing import Final from pydantic import BaseModel, ConfigDict, Field, ValidationError -from tests.integration._support.ordering import TIMINGS, CaseTiming - class ShardingOptions(BaseModel): model_config = ConfigDict(frozen=True) directory: Path shards: int = Field(gt=0) - timings_output: Path | None = None class ShardExecution(BaseModel): @@ -31,16 +28,6 @@ class ShardExecution(BaseModel): shard_index: int complete: bool exitstatus: int - timings: tuple[CaseTiming, ...] = () - - -def merged_timings(executions: tuple[ShardExecution, ...]) -> tuple[CaseTiming, ...]: - timings: Final = tuple(chain.from_iterable(execution.timings for execution in executions)) - measured: Final = Counter(timing.nodeid for timing in timings) - selected: Final = Counter(chain.from_iterable(execution.collected for execution in executions)) - if measured != selected or any(amount != 1 for amount in measured.values()): - return () - return tuple(sorted(timings, key=lambda timing: timing.nodeid)) def shard_errors(executions: tuple[ShardExecution, ...], count: int) -> tuple[str, ...]: @@ -88,7 +75,6 @@ def main() -> int: parser: Final = argparse.ArgumentParser() parser.add_argument("directory", type=Path) parser.add_argument("--shards", required=True, type=int) - parser.add_argument("--timings-output", type=Path) options: Final = ShardingOptions.model_validate(vars(parser.parse_args())) paths: Final = tuple(sorted(options.directory.glob("*/execution.json"))) try: @@ -100,9 +86,6 @@ def main() -> int: if errors: sys.stderr.write("\n".join(errors) + "\n") return 1 - if options.timings_output is not None: - options.timings_output.parent.mkdir(parents=True, exist_ok=True) - options.timings_output.write_bytes(TIMINGS.dump_json(merged_timings(executions))) sys.stdout.write( f"Verified {len(executions[0].inventory)} integration cases across {options.shards} isolated shards\n" ) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index abb4a94c880..280a828e1d4 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -16,7 +16,7 @@ from redis import Redis from tests.integration._support.client import Gateway, eventually, gateway_from_environment from tests.integration._support.generation import LIFECYCLE_SETTINGS from tests.integration._support.manifest import OWNED_DIRECTORIES -from tests.integration._support.ordering import CaseTiming, case_shards, collected_case, fixture_groups, parse_timings +from tests.integration._support.ordering import collected_case, fixture_groups from tests.integration._support.provider import SharedProvider, shared_provider from tests.integration._support.routing import RoutingPlugin from tests.integration.run import GITHUB_FILES @@ -25,7 +25,8 @@ COLLECTED: Final = pytest.StashKey[tuple[str, ...]]() INVENTORY: Final = pytest.StashKey[tuple[str, ...]]() REPORTS: Final = pytest.StashKey[list[pytest.TestReport]]() WORKER_INVENTORY: Final = TypeAdapter(tuple[str, ...]) -TIMING_PATH: Final[TypeAdapter[Path | None]] = TypeAdapter(Path | None) +SHARD_PATH: Final[TypeAdapter[Path | None]] = TypeAdapter(Path | None) +SHARD_FILES: Final = pytest.StashKey[frozenset[str] | None]() INTEGER_OPTION: Final = TypeAdapter(int) @@ -48,13 +49,14 @@ def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption("--integration-order-seed", type=int, default=0) parser.addoption("--integration-shard-count", type=int, default=1) parser.addoption("--integration-shard-index", type=int, default=0) - parser.addoption("--integration-timings", type=Path) + parser.addoption("--integration-shard-files", type=Path) def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line("markers", "integration: owned real-service integration contracts") config.addinivalue_line("markers", "covers(*ids): legacy contract IDs kept for existing tests, not enforced") config.stash[REPORTS] = [] + config.stash[SHARD_FILES] = _shard_files(SHARD_PATH.validate_python(config.getoption("integration_shard_files"))) config.pluginmanager.register(IntegrationReportPlugin(config)) if os.environ.get("INTEGRATION_ROUTING"): config.pluginmanager.register(RoutingPlugin(config)) @@ -100,13 +102,16 @@ def _order_key(seed: int, nodeid: str, group: str = "") -> tuple[bytes, bytes, b return _digest(seed, nodeid.split("::", 1)[0]), _digest(seed, group or nodeid), _digest(seed, nodeid) -def _timing_content(path: Path | None) -> str: +def _shard_files(path: Path | None) -> frozenset[str] | None: if path is None: - return "[]" + return None try: - return path.read_text() - except (OSError, UnicodeError): - return "[]" + files: Final = tuple(path.read_text().splitlines()) + except (OSError, UnicodeError) as error: + raise pytest.UsageError(f"Cannot read integration shard selection: {error}") from error + if len(frozenset(files)) != len(files) or any(not file for file in files): + raise pytest.UsageError("Integration shard selection contains duplicate or blank files") + return frozenset(files) @pytest.hookimpl(wrapper=True) @@ -134,13 +139,15 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item index: Final = INTEGER_OPTION.validate_python(config.getoption("integration_shard_index")) if count < 1 or not 0 <= index < count: raise pytest.UsageError("Integration shard index must be within the positive shard count") - if count > 1: - timing_path: Final = TIMING_PATH.validate_python(config.getoption("integration_timings")) - shards: Final = case_shards( - tuple(collected_case(item) for item in owned), count, parse_timings(_timing_content(timing_path)) - ) - selected: Final = tuple(item for item in items if item not in owned or item.nodeid in shards[index]) - deselected: Final = tuple(item for item in owned if item.nodeid not in shards[index]) + files: Final = config.stash[SHARD_FILES] + if count > 1 and files is None: + raise pytest.UsageError("Parallel integration runs require CircleCI's shard file selection") + if files is not None: + available: Final = frozenset(item.nodeid.split("::", 1)[0] for item in owned) + if files - available: + raise pytest.UsageError(f"Integration shard selected uncollected files: {sorted(files - available)}") + selected: Final = tuple(item for item in items if item not in owned or item.nodeid.split("::", 1)[0] in files) + deselected: Final = tuple(item for item in owned if item.nodeid.split("::", 1)[0] not in files) config.hook.pytest_deselected(items=deselected) items[:] = selected config.stash[COLLECTED] = tuple(item.nodeid for item in items if item in owned) @@ -154,15 +161,13 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: return collected: Final = session.config.stash.get(COLLECTED, ()) reports: Final = tuple(report for report in session.config.stash[REPORTS] if report.nodeid in collected) - timings: Final = tuple( - CaseTiming(nodeid=nodeid, seconds=sum(report.duration for report in reports if report.nodeid == nodeid)) - for nodeid in collected - ) + inventory: Final = session.config.stash.get(INVENTORY, collected) + empty_shard: Final = session.config.stash[SHARD_FILES] == frozenset() and bool(inventory) passed: Final = tuple(report.nodeid for report in reports if report.when == "call" and report.passed) skipped: Final = tuple(report.nodeid for report in reports if report.skipped) complete: Final = ( - exitstatus == 0 - and bool(collected) + (exitstatus == 0 or (empty_shard and exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED)) + and (bool(collected) or empty_shard) and sorted(collected) == sorted(passed + skipped) and not any(report.failed for report in reports) ) @@ -172,14 +177,13 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: json.dumps( { "collected": collected, - "inventory": session.config.stash.get(INVENTORY, collected), + "inventory": inventory, "shard_count": session.config.getoption("integration_shard_count"), "shard_index": session.config.getoption("integration_shard_index"), "passed": passed, "skipped": skipped, - "timings": tuple({"nodeid": timing.nodeid, "seconds": timing.seconds} for timing in timings), "complete": complete, - "exitstatus": exitstatus, + "exitstatus": 0 if complete else exitstatus, "hypothesis_version": version("hypothesis"), "hypothesis_seed": session.config.getoption("hypothesis_seed"), "order_seed": session.config.getoption("integration_order_seed"), @@ -196,6 +200,8 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: ) if not complete and exitstatus == 0: session.exitstatus = pytest.ExitCode.TESTS_FAILED + if complete and empty_shard: + session.exitstatus = pytest.ExitCode.OK @pytest.fixture diff --git a/tests/integration/database/test_harness_lifecycle.py b/tests/integration/database/test_harness_lifecycle.py index 63e3cba7363..91688eef263 100644 --- a/tests/integration/database/test_harness_lifecycle.py +++ b/tests/integration/database/test_harness_lifecycle.py @@ -16,6 +16,7 @@ from contextlib import AbstractContextManager, ExitStack from pathlib import Path from queue import SimpleQueue from threading import Event +from tempfile import TemporaryDirectory from types import FrameType from typing import Final, Literal @@ -38,6 +39,7 @@ from tests.integration._support.database_relay import ( held_statement_relay, ) from tests.integration._support.runtime import configure_executor +from tests.integration._support.sharding import ShardExecution, shard_errors class ExitRecord(TypedDict): @@ -282,75 +284,173 @@ def test_seeded_collection_reuses_scoped_fixtures_and_executes_every_case(tmp_pa ) -def test_partitioned_collection_reuses_unparametrized_fixtures_and_executes_every_case(tmp_path: Path) -> None: +@pytest.mark.parametrize("workers", (1, 2)) +@pytest.mark.parametrize("subset", (False, True)) +def test_native_file_selection_reuses_fixtures_and_qualifies_an_empty_shard( + tmp_path: Path, workers: int, subset: bool +) -> None: events: Final = tmp_path / "events.txt" - probe: Final = tmp_path / "test_fixture_probe.py" - probe.write_text( - "import pytest\nfrom pathlib import Path\n" - f"EVENTS = Path({str(events)!r})\n" - "def record(value):\n" - " with EVENTS.open('a') as output:\n" - " output.write(value + '\\n')\n" - "@pytest.fixture(scope='module')\n" - "def shared():\n" - " record('start shared')\n" - " yield 'shared'\n" - " record('stop shared')\n" - "@pytest.mark.parametrize('number', range(4))\n" - "def test_shared(shared, number):\n" - " assert shared == 'shared'\n" - " record('shared ' + str(number))\n" - "@pytest.mark.parametrize('number', range(8))\n" - "def test_plain(number):\n" - " record('plain ' + str(number))\n" + root: Final = Path(__file__).resolve().parents[3] + with TemporaryDirectory(prefix="native-selection-", dir=Path(__file__).parent) as directory: + probe: Final = Path(directory) / "test_fixture_probe.py" + plain: Final = Path(directory) / "test_plain_probe.py" + probe.write_text( + "import pytest\nfrom pathlib import Path\n" + f"EVENTS = Path({str(events)!r})\n" + "def record(value):\n" + " with EVENTS.open('a') as output:\n" + " output.write(value + '\\n')\n" + "@pytest.fixture(scope='module')\n" + "def shared():\n" + " record('start shared')\n" + " yield 'shared'\n" + " record('stop shared')\n" + "@pytest.mark.parametrize('number', range(4))\n" + "@pytest.mark.xdist_group('shared')\n" + "def test_shared(shared, number):\n" + " assert shared == 'shared'\n" + " record('shared ' + str(number))\n" + ) + plain.write_text( + "import pytest\nfrom pathlib import Path\n" + f"EVENTS = Path({str(events)!r})\n" + "@pytest.mark.parametrize('number', range(8))\n" + "def test_plain(number):\n" + " with EVENTS.open('a') as output:\n" + " output.write('plain ' + str(number) + '\\n')\n" + "def test_existing_skip():\n" + " pytest.skip('retained skip identity')\n" + ) + selections: Final = (str(probe.relative_to(root)), str(plain.relative_to(root)), "") + candidates: Final = ( + (f"{selections[0]}::test_shared[1]", f"{selections[1]}::test_existing_skip") if subset else selections[:2] + ) + listed: Final = subprocess.run( + (sys.executable, "-P", "tests/integration/run.py", "database", "--list", *candidates), + cwd=root, + capture_output=True, + text=True, + timeout=15, + check=False, + ) + assert listed.returncode == 0, listed.stdout + listed.stderr + assert tuple(listed.stdout.splitlines()) == tuple(sorted(selections[:2])) + outputs: Final = tuple(tmp_path / str(index) for index in range(3)) + files: Final = tuple(tmp_path / f"selection-{index}.txt" for index in range(3)) + for selection, path in zip(selections, files, strict=True): + path.write_text(selection + "\n" if selection else "") + results: Final = tuple( + subprocess.run( + ( + sys.executable, + "-P", + "-m", + "pytest", + *candidates, + "--noconftest", + "-p", + "tests.integration.conftest", + "-o", + "addopts=", + "--integration-shard-count=3", + f"--integration-shard-index={index}", + f"--integration-shard-files={files[index]}", + "--integration-order-seed=1848224111", + "--hypothesis-seed=4106601", + "--timeout=15", + *(("-n", str(workers), "--dist=loadgroup") if workers > 1 else ()), + "-q", + ), + cwd=root, + env={ + **{ + name: value + for name, value in os.environ.items() + if name + not in ( + "INTEGRATION_RESULTS_DIR", + "INTEGRATION_ROUTING", + "GITHUB_ACTIONS", + "PYTEST_XDIST_WORKER", + ) + }, + "INTEGRATION_RESULTS_DIR": str(outputs[index]), + "PYTHONPATH": os.pathsep.join((str(root), str(root / "tests"), str(root / "tests/e2e"))), + }, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + for index in range(3) + ) + assert all(result.returncode == 0 for result in results), tuple( + result.stdout + result.stderr for result in results + ) + executions: Final = tuple( + ShardExecution.model_validate_json((output / "execution.json").read_text()) for output in outputs + ) + assert shard_errors(executions, 3) == () + assert len(executions[0].inventory) == (2 if subset else 13) + assert executions[2].collected == () + assert executions[2].complete + assert len(executions[1].skipped) == 1 + assert executions[1].skipped[0].endswith("::test_existing_skip") + observed: Final = tuple(events.read_text().splitlines()) + assert observed.count("start shared") == observed.count("stop shared") == 1 + assert sorted(event for event in observed if event.startswith("shared ")) == ( + ["shared 1"] if subset else [f"shared {index}" for index in range(4)] ) - (tmp_path / "partition_probe.py").write_text( - "from integration._support.ordering import case_shards, collected_case\n" - "def pytest_collection_modifyitems(config, items):\n" - " shards = case_shards(tuple(collected_case(item) for item in items), 3)\n" - " selected = shards[config.getoption('integration_shard_index')]\n" - " items[:] = tuple(item for item in items if item.nodeid in selected)\n" + assert sorted(event for event in observed if event.startswith("plain ")) == ( + [] if subset else [f"plain {index}" for index in range(8)] ) - results: Final = tuple( - subprocess.run( + + +@pytest.mark.parametrize("assignment", ("absent", "missing", "foreign")) +def test_invalid_native_selections_fail_before_any_case_runs( + tmp_path: Path, assignment: Literal["absent", "missing", "foreign"] +) -> None: + root: Final = Path(__file__).resolve().parents[3] + events: Final = tmp_path / "executed.txt" + selection: Final = tmp_path / "node-files.txt" + if assignment == "foreign": + selection.write_text("tests/integration/database/test_unknown.py\n") + with TemporaryDirectory(prefix="invalid-selection-", dir=Path(__file__).parent) as directory: + probe: Final = Path(directory) / "test_probe.py" + probe.write_text(f"from pathlib import Path\ndef test_case():\n Path({str(events)!r}).write_text('ran')\n") + result: Final = subprocess.run( ( sys.executable, "-P", "-m", "pytest", - str(probe), + str(probe.relative_to(root)), + "--noconftest", "-p", "tests.integration.conftest", - "-p", - "partition_probe", - "--integration-shard-count=3", - f"--integration-shard-index={index}", - "--integration-order-seed=1848224111", - "--timeout=15", + "-o", + "addopts=", + "--integration-shard-count=2", + "--integration-shard-index=0", + *((f"--integration-shard-files={selection}",) if assignment != "absent" else ()), "-q", ), + cwd=root, env={ **{ name: value for name, value in os.environ.items() - if name not in ("INTEGRATION_RESULTS_DIR", "INTEGRATION_ROUTING") + if name not in ("INTEGRATION_RESULTS_DIR", "INTEGRATION_ROUTING", "GITHUB_ACTIONS") }, - "PYTHONPATH": os.pathsep.join((str(tmp_path), os.environ.get("PYTHONPATH", ""))), + "PYTHONPATH": os.pathsep.join((str(root), str(root / "tests"), str(root / "tests/e2e"))), }, capture_output=True, text=True, - timeout=30, + timeout=15, check=False, ) - for index in range(3) - ) - assert all(result.returncode == 0 for result in results), tuple(result.stdout + result.stderr for result in results) - observed: Final = tuple(events.read_text().splitlines()) - assert observed.count("start shared") == observed.count("stop shared") == 1 - assert sorted(event for event in observed if event.startswith("shared ")) == [ - f"shared {index}" for index in range(4) - ] - assert sorted(event for event in observed if event.startswith("plain ")) == [f"plain {index}" for index in range(8)] + assert result.returncode == pytest.ExitCode.USAGE_ERROR, result.stdout + result.stderr + assert not events.exists() @pytest.mark.parametrize("entrypoint", ("gateway", "refused")) diff --git a/tests/integration/run.py b/tests/integration/run.py index f89bd15adf9..98b727efce9 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -60,8 +60,8 @@ def main() -> int: parser.add_argument("--workers", type=int, default=int(os.environ.get("INTEGRATION_WORKERS", "1"))) parser.add_argument("--shards", type=int, default=int(os.environ.get("INTEGRATION_CASE_SHARDS", "1"))) parser.add_argument("--shard-index", type=int, default=int(os.environ.get("INTEGRATION_CASE_SHARD_INDEX", "0"))) - parser.add_argument("--timings", type=Path) - parser.add_argument("--list", action="store_true", help="print the group's test files and exit") + parser.add_argument("--shard-files", type=Path) + parser.add_argument("--list", action="store_true", help="print selected candidate files and exit") parser.add_argument("files", nargs="*", help="run only these files, or pytest node ids inside them, of the group") options: Final = parser.parse_intermixed_args() root: Final = Path(__file__).resolve().parents[2] @@ -71,14 +71,14 @@ def main() -> int: for path in sorted((root / "tests/integration" / folder).rglob("test_*.py")) if str(path.relative_to(root)) not in GITHUB_FILES ) - if options.list: - sys.stdout.write("\n".join(group_files) + "\n") - return 0 selection: Final = select(tuple(options.files), group_files) if selection.foreign: parser.error(f"Not in the {options.group} group: {', '.join(selection.foreign)}") if not selection.nodes: parser.error(f"No integration test files selected for {options.group}") + if options.list: + sys.stdout.write("\n".join(sorted({file_of(node) for node in selection.nodes})) + "\n") + return 0 output: Final = options.results.resolve() output.mkdir(parents=True, exist_ok=True) environment: Final = { @@ -109,7 +109,7 @@ def main() -> int: f"--integration-order-seed={options.order_seed}", f"--integration-shard-count={options.shards}", f"--integration-shard-index={options.shard_index}", - *(("--integration-timings", str(options.timings)) if options.timings is not None else ()), + *(("--integration-shard-files", str(options.shard_files)) if options.shard_files is not None else ()), f"--junitxml={output / 'junit.xml'}", "-o", "junit_family=xunit1", diff --git a/tests/unit/integration_support/test_ordering.py b/tests/unit/integration_support/test_ordering.py index 98b08d0849c..775986c6e98 100644 --- a/tests/unit/integration_support/test_ordering.py +++ b/tests/unit/integration_support/test_ordering.py @@ -1,17 +1,12 @@ from __future__ import annotations from itertools import groupby -from types import MappingProxyType from typing import Final -import pytest - from tests.integration._support.ordering import ( CollectedCase, ScopedParameter, - case_shards, fixture_groups, - parse_timings, ) from tests.integration.conftest import _order_key @@ -68,113 +63,3 @@ def test_equal_unhashable_parameters_share_a_group_only_within_their_fixture_own assert groups[first.nodeid] != groups[other_owner.nodeid] assert groups[first.nodeid] != groups[other_file.nodeid] assert groups[plain.nodeid] == "" - - -def test_shards_balance_cases_without_splitting_a_scoped_configuration() -> None: - cases: Final = tuple( - CollectedCase( - f"test_slots.py::test_route[{index}]", - (ScopedParameter("module:slots", "rig", index // 4),), - ("module:slots:extra",) if index % 2 else (), - ) - for index in range(20) - ) - plain: Final = tuple(CollectedCase(f"test_slots.py::test_plain[{index}]") for index in range(7)) - shards: Final = case_shards(cases + plain, 3) - assert frozenset.union(*shards) == frozenset(case.nodeid for case in cases + plain) - assert sum(len(shard) for shard in shards) == len(cases + plain) - assert max(map(len, shards)) - min(map(len, shards)) <= 1 - for _, group in groupby(cases, key=lambda case: case.parameters[0].value): - configuration: Final = frozenset(case.nodeid for case in group) - assert sum(configuration <= shard for shard in shards) == 1 - assert shards == case_shards(tuple(reversed(cases + plain)), 3) - - -def test_shards_reuse_unparametrized_fixtures_and_keep_different_results_separate() -> None: - shared: Final = tuple( - CollectedCase(f"test_sockets.py::test_override[{index}]", fixtures=("module:test_sockets.py:override",)) - for index in range(4) - ) - other: Final = CollectedCase("test_sockets.py::test_default", fixtures=("module:test_sockets.py:default",)) - plain: Final = tuple(CollectedCase(f"test_other.py::test_plain[{index}]") for index in range(7)) - shards: Final = case_shards((*shared, other, *plain), 3) - configuration: Final = frozenset(case.nodeid for case in shared) - assert sum(configuration <= shard for shard in shards) == 1 - assert sorted(map(len, shards)) == [4, 4, 4] - assert frozenset.union(*shards) == frozenset(case.nodeid for case in (*shared, other, *plain)) - groups: Final = fixture_groups((*shared, other)) - assert len({groups[case.nodeid] for case in shared}) == 1 - assert groups[other.nodeid] != groups[shared[0].nodeid] - assert shards == case_shards(tuple(reversed((*shared, other, *plain))), 3) - - -def test_an_oversized_unparametrized_fixture_is_split_into_bounded_groups() -> None: - cases: Final = tuple( - CollectedCase(f"test_sockets.py::test_route[{index:02}]", fixtures=("module:test_sockets.py:owned",)) - for index in range(20) - ) - shards: Final = case_shards(cases, 4) - assert tuple(map(len, shards)) == (5, 5, 5, 5) - assert frozenset.union(*shards) == frozenset(case.nodeid for case in cases) - assert sum(len(shard) for shard in shards) == len(cases) - - -def test_measured_cost_splits_an_expensive_shared_fixture_without_losing_cases() -> None: - expensive: Final = tuple( - CollectedCase(f"test_tracing.py::test_route[{index}]", fixtures=("module:test_tracing.py:gateway",)) - for index in range(4) - ) - cheap: Final = tuple(CollectedCase(f"test_other.py::test_route[{index}]") for index in range(8)) - cases: Final = (*expensive, *cheap) - durations: Final = MappingProxyType({case.nodeid: 40.0 if case in expensive else 1.0 for case in cases}) - shards: Final = case_shards(cases, 4, durations) - assert sorted(sum(durations[nodeid] for nodeid in shard) for shard in shards) == [42.0] * 4 - assert frozenset.union(*shards) == frozenset(case.nodeid for case in cases) - assert sum(len(shard) for shard in shards) == len(cases) - assert shards == case_shards(tuple(reversed(cases)), 4, durations) - - -@pytest.mark.parametrize( - "content", - ( - "invalid", - "{}", - '[{"nodeid":"case","seconds":-1}]', - '[{"nodeid":"case","seconds":"1"}]', - '[{"nodeid":"case","seconds":NaN}]', - '[{"nodeid":"case","seconds":Infinity}]', - '[{"nodeid":"case","seconds":1},{"nodeid":"case","seconds":2}]', - ), -) -def test_invalid_timing_cache_preserves_the_complete_cold_partition(content: str) -> None: - cases: Final = tuple(CollectedCase(f"test_owned.py::test_route[{index}]") for index in range(9)) - durations: Final = parse_timings(content) - assert not durations - assert case_shards(cases, 3, durations) == case_shards(cases, 3) - - -def test_timing_cache_keeps_zero_duration_skips_and_assigns_new_cases() -> None: - durations: Final = parse_timings('[{"nodeid":"skip","seconds":0},{"nodeid":"old","seconds":12}]') - cases: Final = (CollectedCase("skip"), CollectedCase("new")) - shards: Final = case_shards(cases, 2, durations) - assert durations["skip"] > 0 - assert frozenset.union(*shards) == frozenset({"skip", "new"}) - assert tuple(map(len, shards)) == (1, 1) - - -def test_measured_cost_keeps_parametrized_fixture_configurations_atomic() -> None: - shared: Final = tuple( - CollectedCase( - f"test_tracing.py::test_route[{index}]", - (ScopedParameter("module:test_tracing.py", "configuration", "shared"),), - ) - for index in range(3) - ) - plain: Final = tuple(CollectedCase(f"test_other.py::test_route[{index}]") for index in range(5)) - cases: Final = (*shared, *plain) - durations: Final = MappingProxyType({case.nodeid: 40.0 if case in shared else 1.0 for case in cases}) - shards: Final = case_shards(cases, 3, durations) - configuration: Final = frozenset(case.nodeid for case in shared) - assert sum(configuration <= shard for shard in shards) == 1 - assert frozenset.union(*shards) == frozenset(case.nodeid for case in cases) - assert sum(len(shard) for shard in shards) == len(cases) diff --git a/tests/unit/integration_support/test_sharding.py b/tests/unit/integration_support/test_sharding.py index 3d3b53b6f51..2317279f2b2 100644 --- a/tests/unit/integration_support/test_sharding.py +++ b/tests/unit/integration_support/test_sharding.py @@ -2,14 +2,14 @@ from __future__ import annotations from collections.abc import Mapping from dataclasses import dataclass +from pathlib import Path from types import MappingProxyType from typing import Final import pytest -from tests.integration._support.ordering import CaseTiming -from tests.integration._support.sharding import ShardExecution, merged_timings, shard_errors -from tests.integration.conftest import COLLECTED, IntegrationReportPlugin +from tests.integration._support.sharding import ShardExecution, shard_errors +from tests.integration.conftest import COLLECTED, IntegrationReportPlugin, _shard_files from tests.integration.conftest import INVENTORY as WORKER_INVENTORY INVENTORY: Final = ("test_owned.py::test_first", "test_owned.py::test_skipped", "test_owned.py::test_last") @@ -46,10 +46,34 @@ def test_shard_qualification_retains_passes_and_skip_identities() -> None: assert shard_errors(executions, 2) == () +def test_a_selected_subset_can_leave_a_shard_empty_without_losing_the_inventory() -> None: + executions: Final = (_execution(0, INVENTORY), _execution(1, ())) + assert shard_errors(executions, 2) == () + assert shard_errors((_execution(0, INVENTORY[:1]), _execution(1, ())), 2) + + +def test_an_empty_assignment_is_distinct_from_no_assignment_or_a_missing_file(tmp_path: Path) -> None: + assignment: Final = tmp_path / "node-files.txt" + assert _shard_files(None) is None + with pytest.raises(pytest.UsageError, match="Cannot read"): + _shard_files(assignment) + assignment.write_text("") + assert _shard_files(assignment) == frozenset() + + +@pytest.mark.parametrize("content", ("test_owned.py\ntest_owned.py\n", "test_owned.py\n\n", "\n")) +def test_an_invalid_assignment_is_rejected_instead_of_running_extra_tests(tmp_path: Path, content: str) -> None: + assignment: Final = tmp_path / "node-files.txt" + assignment.write_text(content) + with pytest.raises(pytest.UsageError, match="duplicate or blank"): + _shard_files(assignment) + + @pytest.mark.parametrize( "executions", ( (), + (_execution(0, (), inventory=()), _execution(1, (), inventory=())), (_execution(0, INVENTORY),), (_execution(0, INVENTORY[:2]), _execution(0, INVENTORY[2:])), (_execution(0, INVENTORY[:1]), _execution(1, INVENTORY[2:])), @@ -70,34 +94,6 @@ def test_shard_qualification_rejects_lost_duplicate_foreign_or_failed_cases( assert shard_errors(executions, 2) -def test_timing_snapshot_includes_every_executed_pass_and_skip_once() -> None: - first: Final = CaseTiming(nodeid=INVENTORY[0], seconds=40.0) - skipped: Final = CaseTiming(nodeid=INVENTORY[1], seconds=0.0) - last: Final = CaseTiming(nodeid=INVENTORY[2], seconds=1.0) - executions: Final = ( - _execution(0, INVENTORY[:2], skipped=INVENTORY[1:2]).model_copy(update={"timings": (first, skipped)}), - _execution(1, INVENTORY[2:]).model_copy(update={"timings": (last,)}), - ) - assert shard_errors(executions, 2) == () - assert merged_timings(executions) == tuple(sorted((first, skipped, last), key=lambda timing: timing.nodeid)) - - -@pytest.mark.parametrize( - "timings", - ( - (), - (CaseTiming(nodeid=INVENTORY[0], seconds=1.0),), - (CaseTiming(nodeid=INVENTORY[0], seconds=1.0),) * 2, - (CaseTiming(nodeid="foreign", seconds=1.0), CaseTiming(nodeid=INVENTORY[1], seconds=0.0)), - ), -) -def test_incomplete_duplicate_or_foreign_measurements_cannot_seed_the_timing_cache( - timings: tuple[CaseTiming, ...], -) -> None: - execution: Final = _execution(0, INVENTORY[:2]).model_copy(update={"timings": timings}) - assert merged_timings((execution,)) == () - - @dataclass(frozen=True, slots=True) class WorkerResult: workeroutput: Mapping[str, object] From 3981228d7038383fcc7e998e2561ef124f8c878c Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 13:35:49 -0700 Subject: [PATCH 11/40] refactor(anthropic): move raw SSE stream assembly helpers out of proxy guardrails (#45749) * refactor(anthropic): move passthrough SSE stream assembly into litellm/llms Part of LIT-9423 * refactor(anthropic): move raw SSE stream assembly helpers out of proxy guardrails Part of LIT-9423 * refactor(anthropic): give the anthropic_sse re-exports reasoned noqa codes Part of LIT-9423 * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * test(anthropic): cover the raw SSE helpers in stream_assembly Part of LIT-9423 --- litellm/integrations/otel/model/request_io.py | 2 +- .../litellm_core_utils/served_output_texts.py | 5 +- .../anthropic/pass_through/stream_assembly.py | 124 ++++++++++++++- litellm/proxy/guardrails/anthropic_sse.py | 136 ++--------------- .../pass_through/test_stream_assembly.py | 144 ++++++++++++++++++ 5 files changed, 284 insertions(+), 127 deletions(-) create mode 100644 tests/unit/llms/anthropic/pass_through/test_stream_assembly.py diff --git a/litellm/integrations/otel/model/request_io.py b/litellm/integrations/otel/model/request_io.py index 4131f3de6da..287d28a0d17 100644 --- a/litellm/integrations/otel/model/request_io.py +++ b/litellm/integrations/otel/model/request_io.py @@ -6,7 +6,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.integrations.otel.mappers.utils import json_or_none -from litellm.proxy.guardrails.anthropic_sse import assemble_anthropic_sse_stream, is_raw_sse_stream +from litellm.llms.anthropic.pass_through.stream_assembly import assemble_anthropic_sse_stream, is_raw_sse_stream from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse from litellm.types.utils import ModelResponse, ModelResponseStream diff --git a/litellm/litellm_core_utils/served_output_texts.py b/litellm/litellm_core_utils/served_output_texts.py index 2788720448b..ce9a568ea72 100644 --- a/litellm/litellm_core_utils/served_output_texts.py +++ b/litellm/litellm_core_utils/served_output_texts.py @@ -110,7 +110,10 @@ def served_output_texts(response: object) -> ServedTexts | None: def served_stream_output_texts(chunks: Sequence[object]) -> ServedTexts | None: if chunks and all(isinstance(chunk, ModelResponseStream) for chunk in chunks): return _chat_stream_texts(tuple(chunk for chunk in chunks if isinstance(chunk, ModelResponseStream))) - from litellm.proxy.guardrails.anthropic_sse import assemble_anthropic_sse_stream, is_anthropic_sse_stream + from litellm.llms.anthropic.pass_through.stream_assembly import ( + assemble_anthropic_sse_stream, + is_anthropic_sse_stream, + ) if not is_anthropic_sse_stream(chunks): return None diff --git a/litellm/llms/anthropic/pass_through/stream_assembly.py b/litellm/llms/anthropic/pass_through/stream_assembly.py index 76d1896ac32..89961bbe255 100644 --- a/litellm/llms/anthropic/pass_through/stream_assembly.py +++ b/litellm/llms/anthropic/pass_through/stream_assembly.py @@ -1,5 +1,6 @@ +import codecs import json -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import Final import litellm @@ -395,3 +396,124 @@ def build_usage_only_response_from_chunks( ], usage=usage_obj, ) + + +_ANTHROPIC_EVENT_TYPES: Final = frozenset( + { + "message_start", + "message_delta", + "message_stop", + "content_block_start", + "content_block_delta", + "content_block_stop", + "ping", + "error", + } +) + + +def is_raw_sse_stream(all_chunks: Sequence[object]) -> bool: + return any(isinstance(chunk, (str, bytes)) for chunk in all_chunks) + + +def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None: + raw: Final = b"".join( + chunk if isinstance(chunk, bytes) else chunk.encode("utf-8") + for chunk in all_chunks + if isinstance(chunk, (str, bytes)) + ) + try: + return codecs.getincrementaldecoder("utf-8")().decode(raw, final=False) + except UnicodeDecodeError: + return None + + +def _parsed_sse_events(sse_stream: str) -> tuple[Mapping[str, object], ...]: + return tuple( + event_data + for event in split_sse_chunk_into_events(sse_stream) + if (event_data := extract_sse_data(event)) is not None + ) + + +def _anthropic_message_start(sse_stream: str) -> Mapping[str, object] | None: + return next( + ( + message + for event_data in _parsed_sse_events(sse_stream) + if event_data.get("type") == "message_start" and isinstance(message := event_data.get("message"), dict) + ), + None, + ) + + +def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool: + """Whether raw SSE frames are Anthropic Messages events. + + ``is_raw_sse_stream`` only says the chunks are unparsed bytes, and ``/v1/messages`` is not the + only endpoint that streams those: the Google ``:streamGenerateContent`` route marks its own + stream raw too. Reading its frames as Anthropic ones would refuse the response in a wire format + its client cannot parse, so the surface is decided on the event types actually present. + """ + sse_stream: Final = _joined_sse_stream(all_chunks) + if sse_stream is None: + return False + return any(event.get("type") in _ANTHROPIC_EVENT_TYPES for event in _parsed_sse_events(sse_stream)) + + +def assemble_anthropic_sse_stream( + all_chunks: Sequence[object], *, restore_identity: bool = False +) -> ModelResponse | None: + """Assemble raw Anthropic SSE frames into a ModelResponse. + + ``restore_identity`` stamps the upstream message id and model onto the result, which the + assembler does not carry through. It is off by default so callers that re-emit the assembled + response keep the wire shape they had before this helper was shared. The writes land on a + freshly built object that is unreachable from caller state until returned. + """ + sse_stream: Final = _joined_sse_stream(all_chunks) + if sse_stream is None: + return None + message_start: Final = _anthropic_message_start(sse_stream) + if message_start is None: + return None + model: Final = message_start.get("model") if restore_identity else None + try: + assembled: Final = build_complete_streaming_response( + all_chunks=(sse_stream,), + litellm_logging_obj=None, # pyright: ignore[reportArgumentType] # only forwarded to stream_chunk_builder, which accepts None + model=model if isinstance(model, str) else "", + ) + except Exception: # noqa: BLE001 # stream_chunk_builder re-raises every assembly failure as litellm.APIError + return None + if not isinstance(assembled, ModelResponse): + return None + if not restore_identity: + return assembled + message_id: Final = message_start.get("id") + if isinstance(message_id, str): + assembled.id = message_id + if isinstance(model, str) and model: + assembled.model = model + return assembled + + +def is_sse_error_stream(all_chunks: Sequence[object]) -> bool: + """Whether the buffered stream carries nothing but error frames. + + post_call guardrails run in a chain, so a hook can be handed the terminal error frames an + earlier guardrail emitted when it blocked. Those carry no message to assemble, and replacing + them would hide the refusal the client is owed. Covers both wire forms a guardrail emits: the + Anthropic ``error`` event and the chat-completions ``{"error": ...}`` payload. + """ + if not all(isinstance(chunk, (str, bytes)) for chunk in all_chunks): + # A stream mixing typed chunks with an error frame still carries content to scan, and the + # frames-only join below would drop exactly the part that has to be scanned + return False + sse_stream: Final = _joined_sse_stream(all_chunks) + if sse_stream is None: + return False + events: Final = _parsed_sse_events(sse_stream) + return len(events) > 0 and all( + event.get("type") == "error" or isinstance(event.get("error"), Mapping) for event in events + ) diff --git a/litellm/proxy/guardrails/anthropic_sse.py b/litellm/proxy/guardrails/anthropic_sse.py index 07a5d511117..c3a6f259f95 100644 --- a/litellm/proxy/guardrails/anthropic_sse.py +++ b/litellm/proxy/guardrails/anthropic_sse.py @@ -7,115 +7,19 @@ hook scan such a stream, and re-emit it when the guardrail rewrote the response. from __future__ import annotations -import codecs import json -from collections.abc import Mapping, Sequence from typing import Final -from litellm.types.utils import Choices, ModelResponse - -_ANTHROPIC_EVENT_TYPES: Final = frozenset( - { - "message_start", - "message_delta", - "message_stop", - "content_block_start", - "content_block_delta", - "content_block_stop", - "ping", - "error", - } +from litellm.llms.anthropic.pass_through.stream_assembly import ( + assemble_anthropic_sse_stream as assemble_anthropic_sse_stream, # noqa: PLC0414 # public re-export ) - - -def is_raw_sse_stream(all_chunks: Sequence[object]) -> bool: - return any(isinstance(chunk, (str, bytes)) for chunk in all_chunks) - - -def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None: - raw: Final = b"".join( - chunk if isinstance(chunk, bytes) else chunk.encode("utf-8") - for chunk in all_chunks - if isinstance(chunk, (str, bytes)) - ) - try: - return codecs.getincrementaldecoder("utf-8")().decode(raw, final=False) - except UnicodeDecodeError: - return None - - -def _parsed_sse_events(sse_stream: str) -> tuple[Mapping[str, object], ...]: - from litellm.llms.anthropic.pass_through.stream_assembly import extract_sse_data, split_sse_chunk_into_events - - return tuple( - event_data - for event in split_sse_chunk_into_events(sse_stream) - if (event_data := extract_sse_data(event)) is not None - ) - - -def _anthropic_message_start(sse_stream: str) -> Mapping[str, object] | None: - return next( - ( - message - for event_data in _parsed_sse_events(sse_stream) - if event_data.get("type") == "message_start" and isinstance(message := event_data.get("message"), dict) - ), - None, - ) - - -def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool: - """Whether raw SSE frames are Anthropic Messages events. - - ``is_raw_sse_stream`` only says the chunks are unparsed bytes, and ``/v1/messages`` is not the - only endpoint that streams those: the Google ``:streamGenerateContent`` route marks its own - stream raw too. Reading its frames as Anthropic ones would refuse the response in a wire format - its client cannot parse, so the surface is decided on the event types actually present. - """ - sse_stream: Final = _joined_sse_stream(all_chunks) - if sse_stream is None: - return False - return any(event.get("type") in _ANTHROPIC_EVENT_TYPES for event in _parsed_sse_events(sse_stream)) - - -def assemble_anthropic_sse_stream( - all_chunks: Sequence[object], *, restore_identity: bool = False -) -> ModelResponse | None: - """Assemble raw Anthropic SSE frames into a ModelResponse. - - ``restore_identity`` stamps the upstream message id and model onto the result, which the - assembler does not carry through. It is off by default so callers that re-emit the assembled - response keep the wire shape they had before this helper was shared. The writes land on a - freshly built object that is unreachable from caller state until returned. - """ - from litellm.llms.anthropic.pass_through.stream_assembly import build_complete_streaming_response - - sse_stream: Final = _joined_sse_stream(all_chunks) - if sse_stream is None: - return None - message_start: Final = _anthropic_message_start(sse_stream) - if message_start is None: - return None - model: Final = message_start.get("model") if restore_identity else None - try: - assembled: Final = build_complete_streaming_response( - all_chunks=(sse_stream,), - litellm_logging_obj=None, # pyright: ignore[reportArgumentType] # only forwarded to stream_chunk_builder, which accepts None - model=model if isinstance(model, str) else "", - ) - except Exception: # noqa: BLE001 # stream_chunk_builder re-raises every assembly failure as litellm.APIError - return None - if not isinstance(assembled, ModelResponse): - return None - if not restore_identity: - return assembled - message_id: Final = message_start.get("id") - if isinstance(message_id, str): - assembled.id = message_id - if isinstance(model, str) and model: - assembled.model = model - return assembled +from litellm.llms.anthropic.pass_through.stream_assembly import ( + is_anthropic_sse_stream as is_anthropic_sse_stream, # noqa: PLC0414 # public re-export +) +from litellm.llms.anthropic.pass_through.stream_assembly import ( + is_raw_sse_stream as is_raw_sse_stream, # noqa: PLC0414 # public re-export +) +from litellm.types.utils import Choices, ModelResponse def model_response_text(response: ModelResponse) -> str: @@ -141,25 +45,9 @@ def anthropic_sse_error_frames(message: str) -> tuple[bytes, ...]: ) -def is_sse_error_stream(all_chunks: Sequence[object]) -> bool: - """Whether the buffered stream carries nothing but error frames. - - post_call guardrails run in a chain, so a hook can be handed the terminal error frames an - earlier guardrail emitted when it blocked. Those carry no message to assemble, and replacing - them would hide the refusal the client is owed. Covers both wire forms a guardrail emits: the - Anthropic ``error`` event and the chat-completions ``{"error": ...}`` payload. - """ - if not all(isinstance(chunk, (str, bytes)) for chunk in all_chunks): - # A stream mixing typed chunks with an error frame still carries content to scan, and the - # frames-only join below would drop exactly the part that has to be scanned - return False - sse_stream: Final = _joined_sse_stream(all_chunks) - if sse_stream is None: - return False - events: Final = _parsed_sse_events(sse_stream) - return len(events) > 0 and all( - event.get("type") == "error" or isinstance(event.get("error"), Mapping) for event in events - ) +from litellm.llms.anthropic.pass_through.stream_assembly import ( # noqa: E402 # re-export after the definitions above + is_sse_error_stream as is_sse_error_stream, # noqa: PLC0414 # public re-export +) def anthropic_sse_chunks_from_response(assembled: ModelResponse) -> tuple[bytes, ...]: diff --git a/tests/unit/llms/anthropic/pass_through/test_stream_assembly.py b/tests/unit/llms/anthropic/pass_through/test_stream_assembly.py new file mode 100644 index 00000000000..33f95ccc9ff --- /dev/null +++ b/tests/unit/llms/anthropic/pass_through/test_stream_assembly.py @@ -0,0 +1,144 @@ +import json +from typing import Final + +import pytest + +from litellm.llms.anthropic.pass_through.stream_assembly import ( + assemble_anthropic_sse_stream, + is_anthropic_sse_stream, + is_raw_sse_stream, + is_sse_error_stream, +) +from litellm.types.utils import Choices + + +def _frame(event: str, data: dict[str, object]) -> bytes: + return f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() + + +MESSAGE_ID: Final = "msg_upstream_1" +MODEL: Final = "claude-upstream" + +ANTHROPIC_STREAM: Final = ( + _frame( + "message_start", + { + "type": "message_start", + "message": { + "id": MESSAGE_ID, + "type": "message", + "role": "assistant", + "model": MODEL, + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 3, "output_tokens": 1}, + }, + }, + ), + _frame( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _frame( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Hello"}}, + ), + _frame( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": " there"}}, + ), + _frame("content_block_stop", {"type": "content_block_stop", "index": 0}), + _frame( + "message_delta", + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 2}}, + ), + _frame("message_stop", {"type": "message_stop"}), +) + +GOOGLE_STREAM: Final = (b'data: {"candidates": [{"content": {"parts": [{"text": "Hello"}]}}]}\n\n',) +ANTHROPIC_ERROR: Final = _frame("error", {"type": "error", "error": {"type": "overloaded_error", "message": "busy"}}) +CHAT_ERROR: Final = b'data: {"error": {"message": "blocked by guardrail"}}\n\n' +INVALID_UTF8: Final = (b"\xff\xfe data",) + + +@pytest.mark.parametrize( + ("chunks", "expected"), + [ + ((b"data: {}\n\n",), True), + (("data: {}\n\n",), True), + ((object(), b"data: {}\n\n"), True), + ((object(),), False), + ((), False), + ], +) +def test_is_raw_sse_stream_is_true_only_when_a_chunk_is_str_or_bytes(chunks: tuple[object, ...], expected: bool): + assert is_raw_sse_stream(chunks) is expected + + +@pytest.mark.parametrize( + ("chunks", "expected"), + [ + (ANTHROPIC_STREAM, True), + ((ANTHROPIC_ERROR,), True), + (tuple(chunk.decode() for chunk in ANTHROPIC_STREAM), True), + (GOOGLE_STREAM, False), + (INVALID_UTF8, False), + ((), False), + ], +) +def test_is_anthropic_sse_stream_decides_on_the_event_types_present(chunks: tuple[object, ...], expected: bool): + assert is_anthropic_sse_stream(chunks) is expected + + +def test_assemble_anthropic_sse_stream_joins_the_text_deltas(): + assembled = assemble_anthropic_sse_stream(ANTHROPIC_STREAM) + + assert assembled is not None + choice = assembled.choices[0] + assert isinstance(choice, Choices) + assert choice.message.content == "Hello there" + + +def test_assemble_anthropic_sse_stream_restores_upstream_identity_only_when_asked(): + plain = assemble_anthropic_sse_stream(ANTHROPIC_STREAM) + restored = assemble_anthropic_sse_stream(ANTHROPIC_STREAM, restore_identity=True) + + assert plain is not None and restored is not None + assert plain.id != MESSAGE_ID + assert (restored.id, restored.model) == (MESSAGE_ID, MODEL) + + +@pytest.mark.parametrize( + "chunks", + [GOOGLE_STREAM, INVALID_UTF8, (ANTHROPIC_ERROR,), ()], + ids=["no-message-start", "undecodable", "error-only", "empty"], +) +def test_assemble_anthropic_sse_stream_returns_none_without_a_message_to_build(chunks: tuple[object, ...]): + assert assemble_anthropic_sse_stream(chunks) is None + + +@pytest.mark.parametrize( + ("chunks", "expected"), + [ + ((ANTHROPIC_ERROR,), True), + ((CHAT_ERROR,), True), + ((ANTHROPIC_ERROR, CHAT_ERROR), True), + ((ANTHROPIC_ERROR, ANTHROPIC_STREAM[0]), False), + ((object(), ANTHROPIC_ERROR), False), + (ANTHROPIC_STREAM, False), + (INVALID_UTF8, False), + ((b"\n\n",), False), + ], + ids=[ + "anthropic-error", + "chat-error", + "both-error-forms", + "error-then-message", + "typed-chunk-mixed-in", + "normal-stream", + "undecodable", + "no-events", + ], +) +def test_is_sse_error_stream_is_true_only_for_frames_that_are_all_errors(chunks: tuple[object, ...], expected: bool): + assert is_sse_error_stream(chunks) is expected From dc95de52097e7824ed7d6c9fb1d290453d1e35e1 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:52:32 -0700 Subject: [PATCH 12/40] feat(otel)!: let a team capture_message_content override the global setting for its OTel v2 destination (#44264) * feat(otel): let a team strip captured message content from its own OTel v2 destination Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): split the shared callback option conflict check and sync dashboard API types Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): address review feedback on team capture_message_content Classify capture_message_content as NotSecret in the credential slot registry, inject test exporters through register_exporter_factory instead of patching _exporter_from_spec, type the test helpers, and stop reassigning processor in build_tracer_provider Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel): add integration cells for team capture_message_content Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel): cover weave_otel and newrelic team destinations in capture_message_content cells Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): stop rejecting client-sent capture_message_content at the request body boundary Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(otel): let a team capture_message_content override the global setting in both directions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): keep a team's capture_message_content on its own OTel v2 destination The setting no longer rides the flattened team callback vars, so a sibling destination and a routed provider follow the global mode. Request bodies get the base 400 back, a failure-only entry rejects the setting, and entries for one backend must agree Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel): type the capture_message_content callback validation tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel): type the remaining capture_message_content test signatures Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel): mark the new capture_message_content test locals Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): default tenant capture to callback_settings.otel and keep CaptureMessageContent formatting bare Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): keep CaptureMessageContent a plain str like main The str-to-Enum switch changed the public type: the config field's type, repr and model_dump, a constructor that now raised on unknown values, and unknown env values coerced to no_content. CaptureMessageContent is the plain str subclass main ships again, the field is a str, and a captures_span_content helper replaces the enum property. A test pins that a published preset's capture mode stays the default for an unowned destination, which is what main forwards. * test(otel): drop the module-level CaptureMessageContent import the rewritten test no longer uses * fix(lint): keep the merged branch within main's type budgets Freeze the capture-mode settings model (LIT015) and read stored logging entries through one typed sequence helper, so the conflicting-capture checks add no unknown types. * fix(otel): reject a destination capture_message_content while OTel v2 is off The legacy OTel callbacks never read the per-destination setting, so a team could register no_content and still export prompts and replies under the global capture mode. Registration now refuses the var unless the proxy runs with LITELLM_OTEL_V2=true, the same rule the Arize transport and New Relic routing vars follow. * test(otel): type the OTel v2 fixtures and the v2-off regression test * fix(otel): accept only no_content and span_only on a team or key destination OTel v2 writes captured content only to span attributes, so event_only behaved like no_content and span_and_event like span_only on a destination. The proxy-wide setting keeps all four values * docs(otel): say an omitted capture setting follows the proxy's own callback for that backend * test(otel): describe the general capture fallback in the unowned destination test --------- Co-authored-by: mrinal Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yucheng-berri Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- litellm/integrations/otel/README.md | 4 +- litellm/integrations/otel/langfuse_logger.py | 6 + litellm/integrations/otel/logger.py | 17 +- litellm/integrations/otel/model/config.py | 59 +- .../integrations/otel/model/destination.py | 19 +- .../integrations/otel/plumbing/providers.py | 187 ++++- litellm/integrations/otel/plumbing/routing.py | 3 +- .../integrations/otel/presets/destinations.py | 2 + litellm/integrations/otel/presets/newrelic.py | 7 +- .../initialize_dynamic_callback_params.py | 8 + litellm/proxy/_types.py | 6 +- .../callback_config_validation.py | 81 +- litellm/proxy/litellm_pre_call_utils.py | 39 +- .../team_callback_endpoints.py | 13 + litellm/types/utils.py | 22 + tests/integration/_support/otlp_sink.py | 5 +- .../test_otel_team_capture_message_content.py | 720 ++++++++++++++++++ .../integrations/otel/test_langfuse_logger.py | 34 +- .../otel/test_otel_v2_destinations.py | 582 +++++++++++++- .../integrations/otel/test_otel_v2_logger.py | 59 +- .../otel/test_otel_v2_sources_of_truth.py | 13 + .../test_callback_config_validation.py | 115 +++ .../test_team_callback_endpoints.py | 60 ++ tests/unit/types/test_utils.py | 40 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 3 + 25 files changed, 2019 insertions(+), 85 deletions(-) create mode 100644 tests/integration/observability/test_otel_team_capture_message_content.py create mode 100644 tests/unit/types/test_utils.py diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md index 38f96f954f1..4b963ca3415 100644 --- a/litellm/integrations/otel/README.md +++ b/litellm/integrations/otel/README.md @@ -295,7 +295,9 @@ nothing here imports outside it: - [`config.py`](./model/config.py) — `OpenTelemetryV2Config`, a pydantic-settings model that reads `OTEL_*` / `LITELLM_OTEL_*` env vars, plus the feature gate. `capture_span_content` gates whether prompt/response bodies may be written as - span attributes; it defaults **off** (`no_content`). The Baggage allowlists are + span attributes; it defaults **off** (`no_content`). A team or key can override + the global mode per OTel v2 destination with `capture_message_content`; omitted + values follow the proxy's own callback for that backend, else the global mode. The Baggage allowlists are configurable, not hard-coded: set `LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS` / `LITELLM_OTEL_BAGGAGE_METADATA_KEYS` / `LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS` (comma-separated) as env vars, or diff --git a/litellm/integrations/otel/langfuse_logger.py b/litellm/integrations/otel/langfuse_logger.py index d029b153c52..8f6a7d5eb51 100644 --- a/litellm/integrations/otel/langfuse_logger.py +++ b/litellm/integrations/otel/langfuse_logger.py @@ -51,6 +51,10 @@ class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2): response: "AsyncIterator[ModelResponseStream]", request_data: Mapping[str, object], ) -> "AsyncGenerator[ModelResponseStream, None]": + if not self._capture_span_content(): + async for chunk in response: + yield chunk + return relayed: Final[list[ModelResponseStream]] = [] # mutable-ok: relayed as they arrive, assembled at end of stream async for chunk in response: relayed.append(chunk) @@ -58,6 +62,8 @@ class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2): self._stamp_root_io(request_data, lambda: stream_output(tuple(relayed), request_data)) def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None: + if not self._capture_span_content(): + return root: Final = request_root_span() if root is None or not root.is_recording(): return diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 498ebe655da..d8dccc39c13 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -58,6 +58,7 @@ from litellm.integrations.otel.plumbing.context import ( is_recordable_span, mcp_message_transport_span, post_response_root, + request_destinations, request_root_http_route, request_root_span, resolve_internal_call_span_context, @@ -82,6 +83,7 @@ from litellm.integrations.otel.plumbing.providers import ( resolve_meter_provider, ) from litellm.integrations.otel.plumbing.routing import TenantTracerCache +from litellm.types.utils import captures_span_content if TYPE_CHECKING: from opentelemetry.metrics import MeterProvider @@ -254,6 +256,11 @@ class OpenTelemetryV2(CustomLogger): return None return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME), provider.resource) + def _capture_span_content(self) -> bool: + return self.config.capture_span_content or any( + captures_span_content(destination.capture_message_content) for destination in request_destinations() + ) + # ====================================================================== # # Proxy global registration # ====================================================================== # @@ -433,7 +440,7 @@ class OpenTelemetryV2(CustomLogger): return False payload: Final = cast("StandardLoggingPayload", raw_payload) data: Final = MCPToolCallSpanData.from_standard_logging_payload( - payload, capture_content=self.config.capture_span_content + payload, capture_content=self._capture_span_content() ) # A stray LLM carrier from a ``pre_call`` that mis-fired for this id would # otherwise linger until evicted; drop it so it's neither leaked nor closed @@ -477,7 +484,7 @@ class OpenTelemetryV2(CustomLogger): return False payload: Final = cast("StandardLoggingPayload", raw_payload) data: Final = MCPListToolsSpanData.from_standard_logging_payload( - payload, capture_content=self.config.capture_span_content + payload, capture_content=self._capture_span_content() ) if data.identity.call_id: self._release_carrier(self._open_llm_calls.pop(data.identity.call_id, None)) @@ -569,7 +576,7 @@ class OpenTelemetryV2(CustomLogger): return None data: Final = LLMCallSpanData.from_standard_logging_payload( payload, - capture_content=self.config.capture_span_content, + capture_content=self._capture_span_content(), time_to_first_chunk_seconds=call.time_to_first_chunk_seconds, request_route=request_root_http_route(), request_purpose=call.purpose, @@ -1076,6 +1083,6 @@ def build_otel_v2_logger( def _logger_class(config: OpenTelemetryV2Config) -> type[OpenTelemetryV2]: if "langfuse" not in config.mapper_names: return OpenTelemetryV2 - from litellm.integrations.otel.langfuse_logger import LangfuseContentOpenTelemetryV2, LangfuseOpenTelemetryV2 + from litellm.integrations.otel.langfuse_logger import LangfuseContentOpenTelemetryV2 - return LangfuseContentOpenTelemetryV2 if config.capture_span_content else LangfuseOpenTelemetryV2 + return LangfuseContentOpenTelemetryV2 diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index 3cf49ea2f11..55ef28bf470 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -16,19 +16,17 @@ from litellm.integrations.otel.model.baggage import ( ) from litellm.integrations.otel.model.spans import POSTGRESQL, db_system from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.utils import OtelSpanScope +from litellm.types.utils import ( + CAPTURE_MESSAGE_CONTENT_VALUES, + CaptureMessageContent, + OtelSpanScope, + captures_span_content, +) #: Master feature-flag env var. The logger is inert until this is truthy. OTEL_V2_ENV: Final = "LITELLM_OTEL_V2" -class CaptureMessageContent(str): - NO_CONTENT = "no_content" - SPAN_ONLY = "span_only" - EVENT_ONLY = "event_only" - SPAN_AND_EVENT = "span_and_event" - - class ExporterOwner(str, Enum): """The preset that contributed an exporter. Values match the callback names in ``presets.PRESET_BY_CALLBACK`` so per-request dynamic-credential routing @@ -297,9 +295,7 @@ class OpenTelemetryV2Config(BaseSettings): here are lower_snake_case; normalizing at the boundary keeps both spellings working and lets every downstream comparison stay exact. """ - if isinstance(value, str): - return value.lower() - return value + return _normalized_capture_message_content(value) @field_validator("langfuse_span_scope", mode="before") @classmethod @@ -371,22 +367,39 @@ class OpenTelemetryV2Config(BaseSettings): @property def capture_span_content(self) -> bool: - """Whether prompt/response content may be stamped as span attributes. - - Defaults off (``no_content``): an operator must opt in before message - bodies leave the process, so a user request can never force its prompt - or completion into the configured backend while capture is disabled. - """ - return self.capture_message_content in ( - CaptureMessageContent.SPAN_ONLY, - CaptureMessageContent.SPAN_AND_EVENT, - ) + """Whether the global setting permits prompt/response span attributes.""" + return captures_span_content(self.capture_message_content) @classmethod def from_env(cls) -> "OpenTelemetryV2Config": return cls() +def _normalized_capture_message_content(value: object) -> object: + if isinstance(value, str): + return value.lower() + return value + + +class _CaptureMessageContentSettings(BaseSettings): + model_config = SettingsConfigDict(extra="ignore", frozen=True) + + capture_message_content: str = Field( + default=CaptureMessageContent.NO_CONTENT, + validation_alias=AliasChoices("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"), + ) + + @field_validator("capture_message_content", mode="before") + @classmethod + def _normalize_capture_message_content(cls, value: object) -> object: + return _normalized_capture_message_content(value) + + +def capture_message_content_from_env() -> str: + """The global capture mode, read without building a settings model that rereads every other env var""" + return _CaptureMessageContentSettings().capture_message_content + + _EXCLUDED_SERVICES_INPUT: Final[TypeAdapter[str | tuple[object, ...]]] = TypeAdapter(str | tuple[object, ...]) @@ -395,6 +408,10 @@ def excluded_db_systems_from(value: object) -> frozenset[str]: return _normalize_excluded_services(excluded_service_names(value)) +def parse_capture_message_content(value: object) -> str | None: + return value if isinstance(value, str) and value in CAPTURE_MESSAGE_CONTENT_VALUES else None + + def excluded_service_names(value: object) -> frozenset[str]: """Read a YAML list or comma-separated string of service names, logging and dropping unusable input so a malformed value cannot stop the OTel logger from being built""" diff --git a/litellm/integrations/otel/model/destination.py b/litellm/integrations/otel/model/destination.py index 56b3123660c..b0c22dea292 100644 --- a/litellm/integrations/otel/model/destination.py +++ b/litellm/integrations/otel/model/destination.py @@ -11,7 +11,7 @@ from urllib.parse import quote from pydantic import ConfigDict, Field from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.utils import OtelSpanScope +from litellm.types.utils import OtelSpanScope, captures_span_content class OtelDestination(LiteLLMBaseModel): @@ -50,6 +50,17 @@ class OtelDestination(LiteLLMBaseModel): default=None, description="Same, for the requests with a failed span in their tree; ``None`` forwards every one.", ) + capture_message_content: str | None = Field( + default=None, + description=( + "An explicit mode overrides the global capture policy for this destination. " + "Omitted follows the global setting." + ), + ) + + def captures_content(self, default: str) -> bool: + setting: Final = self.capture_message_content + return captures_span_content(default if setting is None else setting) def header_string(self) -> str: """Render headers as the ``k=v,k2=v2`` form an ``ExporterSpec`` expects. @@ -64,9 +75,9 @@ class OtelDestination(LiteLLMBaseModel): def cache_key(self) -> tuple[str, tuple[tuple[str, str], ...], tuple[tuple[str, str], ...], str | None]: """Identity for processor reuse, so one destination means one exporter. - ``span_scope`` and the sampling rates are left out on purpose: they decide which - spans reach the processor, not how the processor exports them, so a full and an - ``llm_only`` view of the same account share one exporter. + ``span_scope``, the sampling rates and ``capture_message_content`` are left out on + purpose: they decide which spans reach the processor and what they carry, not how + the processor exports them, so two views of the same account share one exporter. """ return ( self.endpoint, diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 37a8f1951d9..4e7d4f231ed 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -5,7 +5,7 @@ import random import threading import time from collections import OrderedDict -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal @@ -40,8 +40,19 @@ from opentelemetry.util.types import Attributes, AttributeValue from litellm._logging import verbose_logger from litellm._version import version as litellm_version -from litellm.integrations.otel.mappers.langfuse import LANGFUSE_TRACE_NAME -from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.mappers.langfuse import ( + LANGFUSE_OBSERVATION_INPUT, + LANGFUSE_OBSERVATION_OUTPUT, + LANGFUSE_TRACE_NAME, +) +from litellm.integrations.otel.model.config import ( + CaptureMessageContent, + ExporterOwner, + ExporterSpec, + OpenTelemetryV2Config, + capture_message_content_from_env, + captures_span_content, +) from litellm.integrations.otel.model.semconv import ( DB, MCP, @@ -385,6 +396,26 @@ _CAPTURED_HEADER_PREFIXES: Final = ("http.request.header.", "http.response.heade # ``?key=`` query parameter. _URL_KEYS: Final = frozenset({"http.url", "http.target", "url.full"}) _URL_QUERY_KEY: Final = "url.query" +# Prompt and response content, in every vocabulary the mappers write it in once the +# global capture policy collects it: GenAI semconv (MCP tool-call arguments and results +# included), Langfuse, OpenInference's blobs and per-index messages, Langtrace, Weave. +_MESSAGE_CONTENT_KEYS: Final = frozenset( + { + GenAI.INPUT_MESSAGES, + GenAI.OUTPUT_MESSAGES, + GenAI.SYSTEM_INSTRUCTIONS, + GenAI.TOOL_CALL_ARGUMENTS, + GenAI.TOOL_CALL_RESULT, + LANGFUSE_OBSERVATION_INPUT, + LANGFUSE_OBSERVATION_OUTPUT, + "input.value", + "output.value", + "llm.prompts", + "llm.completions", + "weave.output", + } +) +_INDEXED_MESSAGE_PREFIXES: Final = ("llm.input_messages.", "llm.output_messages.") class _SpanView(ReadableSpan): @@ -415,6 +446,19 @@ class _SpanView(ReadableSpan): ) +def _is_message_content(key: str) -> bool: + return key in _MESSAGE_CONTENT_KEYS or key.startswith(_INDEXED_MESSAGE_PREFIXES) + + +def without_message_content(span: ReadableSpan) -> ReadableSpan: + """``span`` without its prompt and response content, everything else left as it was.""" + attributes: Final = span.attributes or _NO_ATTRIBUTES + if not any(_is_message_content(key) for key in attributes): + return span + kept: Final = MappingProxyType({key: value for key, value in attributes.items() if not _is_message_content(key)}) + return _SpanView(span, span.resource, kept, span.events, span.status, parent=span.parent) + + def _is_database_span(attributes: Mapping[str, AttributeValue]) -> bool: return any(key in attributes for key in _DB_SYSTEM_KEYS) @@ -553,7 +597,11 @@ def _tenant_resource(resource: Resource, defaults: Mapping[str, str], extra: Map return filled.merge(Resource(extra)) if extra else filled -def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> ReadableSpan: +def _for_destination( + span: ReadableSpan, + destination: "OtelDestination", + default_capture: str, +) -> ReadableSpan: """The view of ``span`` a tenant destination receives. A span the tenant's own call produced keeps its error text. Every other span is @@ -564,8 +612,10 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read exception it raised and names the operator's guardrail endpoint. Stack traces walk the operator's install and come off every span, as do the headers the operator captures on the server span, whose request side holds the caller's bearer token, - and the query string of the request URL, which can hold the same key. The span - itself stays, so the tenant still gets the whole trace tree. + and the query string of the request URL, which can hold the same key. A + destination whose effective setting disables span content loses the prompt and + response content too. The span itself stays, so the tenant still gets the whole + trace tree. """ extra: Final = destination.resource_attributes defaults: Final = destination.resource_defaults @@ -573,11 +623,12 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read database: Final = _is_database_span(attributes) owned: Final = _is_tenant_owned_span(attributes) unreachable: Final = _guardrail_unreachable(attributes) + redacted: Final = not destination.captures_content(default_capture) kept: Final = MappingProxyType( { key: _without_query(key, value) for key, value in attributes.items() - if _tenant_visible(key, database, owned, unreachable) + if _tenant_visible(key, database, owned, unreachable) and not (redacted and _is_message_content(key)) } ) recorded: Final = span.events @@ -612,6 +663,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): shutdown_drain_seconds: float = _SHUTDOWN_DRAIN_SECONDS, operator_sinks: 'Mapping[_SinkKey, "OtelSpanScope"]' = MappingProxyType({}), excluded_db_systems: frozenset[str] = frozenset(), + default_capture: str = CaptureMessageContent.NO_CONTENT, + default_capture_by_backend: Mapping[str, str] = MappingProxyType({}), pending_drains: int = _MAX_PENDING_DRAINS, drain_pool: _DrainPool | None = None, sampling_draw: Callable[[], float] | None = None, @@ -619,6 +672,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._operator_sinks: Final = operator_sinks self._draw: Final = sampling_draw if sampling_draw is not None else random.random self._excluded_db_systems: Final = excluded_db_systems + self._default_capture: Final = default_capture + self._default_capture_by_backend: Final = default_capture_by_backend self._drain_seconds: Final = shutdown_drain_seconds self._lock: Final = threading.Condition() self._closed = False # guarded by ``_lock``: an unlocked read races the teardown it gates @@ -661,12 +716,22 @@ class TenantFanOutSpanProcessor(SpanProcessor): for held_span, held_destination in self._settle(trace_id, failed): self._forward(held_span, held_destination) + def _default_capture_for(self, destination: "OtelDestination") -> str: + backend: Final = destination.callback_name + return ( + self._default_capture_by_backend.get(backend, self._default_capture) if backend else self._default_capture + ) + def _forward(self, span: ReadableSpan, destination: "OtelDestination") -> None: processor: Final = self._acquire(destination) if processor is None: return try: - processor.on_end(_scoped(_for_destination(span, destination), destination.span_scope)) + processor.on_end( + _scoped( + _for_destination(span, destination, self._default_capture_for(destination)), destination.span_scope + ) + ) except Exception as exc: # noqa: BLE001 # one destination's failure must not cost the others their span verbose_logger.debug("OTel V2 fan-out: forwarding to %s failed: %s", destination.endpoint, exc) finally: @@ -963,14 +1028,15 @@ class _OverriddenBackendFilter(SpanProcessor): Wrapping is the only place this works: ``SynchronousMultiSpanProcessor.on_end`` ignores return values, so a sibling processor can never veto the export. - Under ``additive`` mode nothing is suppressed, so the wrapper passes every span - straight through and the operator keeps its copy. + Under ``additive`` mode spans are not suppressed, though global content policy + still strips message content when disabled. ``scope`` narrows what the exporter receives independently of that: under ``llm_only`` the model-call spans go through as trace roots and the rest of the tree is held back, unless a destination of the request names ``sink``, the account this exporter writes to, with a wider scope: the fan-out then delivers the rest of - the tree there and the model call keeps its place in it. + the tree there and the model call keeps its place in it. A destination naming + ``sink`` sets this exporter's content policy; otherwise it follows global capture. """ def __init__( @@ -979,11 +1045,15 @@ class _OverriddenBackendFilter(SpanProcessor): owner: str | None, scope: "OtelSpanScope" = "full", sink: _SinkKey | None = None, + global_captures: bool = True, + filter_content: bool = True, ) -> None: self._inner: Final = inner self._owner: Final = owner self._scope: Final = scope self._sink: Final = sink + self._global_captures: Final = global_captures + self._filter_content: Final = filter_content def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: self._inner.on_start(span, parent_context) @@ -991,7 +1061,28 @@ class _OverriddenBackendFilter(SpanProcessor): def on_end(self, span: ReadableSpan) -> None: if self._owner in suppressed_backends() or not _in_scope(span, self._scope): return - self._inner.on_end(_scoped(span, self._account_scope())) + scoped: Final = _scoped(span, self._account_scope()) + self._inner.on_end( + without_message_content(scoped) if self._filter_content and not self._account_captures_content() else scoped + ) + + def _account_captures_content(self) -> bool: + """Whether a destination of this request that names ``sink`` captures content. + + The fan-out skips a destination this exporter already writes to, so this copy is + the one that account gets and has to honor the destination's content mode. + """ + explicit: Final = next( + ( + destination.capture_message_content + for destination in request_destinations() + if self._sink is not None + and destination.capture_message_content is not None + and _sink_key(destination.endpoint, destination.headers) == self._sink + ), + None, + ) + return captures_span_content(explicit) if explicit is not None else self._global_captures def _account_scope(self) -> "OtelSpanScope": if self._scope == "full" or self._sink is None: @@ -1010,6 +1101,25 @@ class _OverriddenBackendFilter(SpanProcessor): return self._inner.force_flush(timeout_millis) +class _MessageContentFilter(SpanProcessor): + """Strip prompt and response content from every span before ``inner`` exports it.""" + + def __init__(self, inner: SpanProcessor) -> None: + self._inner: Final = inner + + def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: + self._inner.on_start(span, parent_context) + + def on_end(self, span: ReadableSpan) -> None: + self._inner.on_end(without_message_content(span)) + + def shutdown(self) -> None: + self._inner.shutdown() + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return self._inner.force_flush(timeout_millis) + + def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter: """Build a single exporter from the top-level config fields. @@ -1261,12 +1371,23 @@ def build_resource(config: OpenTelemetryV2Config) -> Resource: return Resource.create(attributes) +def _spec_processor( + spec: ExporterSpec, use_simple_processor: bool | None, strip_message_content: bool +) -> SpanProcessor: + exporting: Final = _processor_for( + _exporter_from_spec(spec), + (spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor), + ) + return _MessageContentFilter(exporting) if strip_message_content else exporting + + def build_tracer_provider( config: OpenTelemetryV2Config, exporter: SpanExporter | None = None, baggage_processor: SpanProcessor | None = None, use_simple_processor: bool | None = None, tenant_overrides: bool = False, + strip_message_content: bool = False, ) -> TracerProvider: """Build the shared :class:`TracerProvider`. @@ -1285,6 +1406,9 @@ def build_tracer_provider( ``config.langfuse_span_scope`` narrows the exporter owned by ``langfuse_otel`` alone; a collector or any other backend in the same config keeps the full tree. + ``strip_message_content`` removes prompt and response content before every + exporter, for a routed provider whose spans can carry content another + destination asked for. """ provider: Final = TracerProvider(resource=build_resource(config)) if baggage_processor is None: @@ -1300,17 +1424,23 @@ def build_tracer_provider( for spec in config.exporters: if spec.requires_headers and not spec.headers: continue - exp = _exporter_from_spec(spec) - processor = _processor_for( - exp, - (spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor), - ) + processor = _spec_processor(spec, use_simple_processor, strip_message_content) owner = spec.owner.value if tenant_overrides and spec.owner is not None else None scope = _operator_scope(config, spec) sink = _sink_key(spec.endpoint, parse_headers(spec.headers)) if _exports_to_the_wire(spec) else None provider.add_span_processor( - _OverriddenBackendFilter(processor, owner, scope, sink) - if owner is not None or scope != "full" + _OverriddenBackendFilter( + processor, + owner, + scope, + sink, + config.capture_span_content, + tenant_overrides, + ) + if owner is not None + or scope != "full" + or sink is not None + or (tenant_overrides and not config.capture_span_content) else processor ) return provider @@ -1340,10 +1470,29 @@ def attach_tenant_fan_out( TenantFanOutSpanProcessor( operator_sinks=operator_sink_scopes(*configs), excluded_db_systems=excluded_db_systems, + default_capture=_operator_capture_mode(configs), + default_capture_by_backend=MappingProxyType(dict(_owned_capture_modes(configs))), ) ) +def _operator_capture_mode(configs: Sequence[OpenTelemetryV2Config]) -> str: + """The mode of the operator's generic ``otel`` callback, so ``callback_settings.otel`` counts as the global default""" + return next( + (config.capture_message_content for config in configs if any(spec.owner is None for spec in config.exporters)), + capture_message_content_from_env(), + ) + + +def _owned_capture_modes( + configs: Sequence[OpenTelemetryV2Config], +) -> Iterator[tuple[str, str]]: + for config in configs: + for spec in config.exporters: + if spec.owner is not None: + yield spec.owner.value, config.capture_message_content + + def deliverable_destinations( destinations: Iterable["OtelDestination"], provider: trace.TracerProvider | None = None, diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py index d7b1cadfc92..eb90b9035c5 100644 --- a/litellm/integrations/otel/plumbing/routing.py +++ b/litellm/integrations/otel/plumbing/routing.py @@ -282,7 +282,8 @@ class TenantTracerCache: self._providers.move_to_end(cache_key) return cached built: Final = build_tracer_provider( - self._routed_config(credential_headers, project_headers, endpoint, service_name) + self._routed_config(credential_headers, project_headers, endpoint, service_name), + strip_message_content=not self._config.capture_span_content, ) self._providers[cache_key] = built return built diff --git a/litellm/integrations/otel/presets/destinations.py b/litellm/integrations/otel/presets/destinations.py index 91341a57634..c4273fbb337 100644 --- a/litellm/integrations/otel/presets/destinations.py +++ b/litellm/integrations/otel/presets/destinations.py @@ -187,6 +187,7 @@ def destination_for( callback_name: str, params: StandardCallbackDynamicParams, service_name: str | None = None, + capture_message_content: str | None = None, ) -> OtelDestination | None: """The destination ``params`` names for ``callback_name``, or ``None``. @@ -217,6 +218,7 @@ def destination_for( callback_name=callback_name, protocol=protocol, span_scope=_span_scope(callback_name, params), + capture_message_content=capture_message_content, success_sampling_rate=success_sampling_rate, error_sampling_rate=error_sampling_rate, ) diff --git a/litellm/integrations/otel/presets/newrelic.py b/litellm/integrations/otel/presets/newrelic.py index 771b3f643c8..9d8d04d8f0d 100644 --- a/litellm/integrations/otel/presets/newrelic.py +++ b/litellm/integrations/otel/presets/newrelic.py @@ -9,6 +9,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict from litellm._logging import verbose_logger from litellm.integrations.otel.model.config import ( + CaptureMessageContent, ExporterOwner, ExporterSpec, OpenTelemetryV2Config, @@ -66,7 +67,11 @@ def newrelic_preset( # New Relic ingests the OTLP GenAI semantic conventions natively. "mapper_names": ensure_mappers(base.mapper_names, "genai"), **( - {"capture_message_content": ("span_only" if settings.record_content else "no_content")} + { + "capture_message_content": ( + CaptureMessageContent.SPAN_ONLY if settings.record_content else CaptureMessageContent.NO_CONTENT + ) + } if settings.record_content is not None else {} ), diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index d2ba17e169a..2efeead6bb2 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -6,6 +6,7 @@ from typing import Any, Final from litellm.types.utils import ( ARIZE_OTLP_PROTOCOLS, + CAPTURE_MESSAGE_CONTENT_VALUES, OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams, @@ -91,6 +92,13 @@ def validate_arize_otlp_protocol_value(value: str) -> None: raise ValueError(f"Invalid arize_otlp_protocol {value!r}: must be one of {sorted(ARIZE_OTLP_PROTOCOLS)}") +def validate_capture_message_content_value(value: str) -> None: + if value not in CAPTURE_MESSAGE_CONTENT_VALUES: + raise ValueError( + f"Invalid capture_message_content {value!r}: must be one of {sorted(CAPTURE_MESSAGE_CONTENT_VALUES)}" + ) + + # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict _supported_callback_params: Final[tuple[str, ...]] = ( "langfuse_public_key", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 0715a4cae97..39892bcc29c 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -33,6 +33,7 @@ from litellm._uuid import uuid from litellm.constants import DEFAULT_STAGGER_WINDOW_SECONDS, MCP_STDIO_ALLOWED_COMMANDS from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( validate_arize_otlp_protocol_value, + validate_capture_message_content_value, validate_langfuse_environment_value, validate_langfuse_span_scope_value, validate_no_callback_env_reference, @@ -79,6 +80,7 @@ from litellm.types.router import AllowedModelRegion, RouterErrors, UpdateRouterC from litellm.types.router_weights import validate_router_settings_dict from litellm.types.secret_managers.main import KeyManagementSystem from litellm.types.utils import ( + CAPTURE_MESSAGE_CONTENT_VAR, AzureSpillover, CallTypes, CostBreakdown, @@ -2202,7 +2204,7 @@ class AddTeamCallback(LiteLLMPydanticObjectBase): @classmethod def validate_callback_vars(cls, values): callback_vars: Final = values.get("callback_vars", {}) - valid_keys: Final = set(StandardCallbackDynamicParams.__annotations__.keys()) + valid_keys: Final = {*StandardCallbackDynamicParams.__annotations__, CAPTURE_MESSAGE_CONTENT_VAR} for key, value in callback_vars.items(): if key not in valid_keys: raise ValueError(f"Invalid callback variable: {key}. Must be one of {valid_keys}") @@ -2214,6 +2216,8 @@ class AddTeamCallback(LiteLLMPydanticObjectBase): validate_langfuse_span_scope_value(callback_vars[key]) if key == "arize_otlp_protocol": validate_arize_otlp_protocol_value(callback_vars[key]) + if key == CAPTURE_MESSAGE_CONTENT_VAR: + validate_capture_message_content_value(callback_vars[key]) return values diff --git a/litellm/proxy/common_utils/callback_config_validation.py b/litellm/proxy/common_utils/callback_config_validation.py index 5b9fb2e3977..4956da853c6 100644 --- a/litellm/proxy/common_utils/callback_config_validation.py +++ b/litellm/proxy/common_utils/callback_config_validation.py @@ -10,6 +10,8 @@ from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Final +from litellm.types.utils import CAPTURE_MESSAGE_CONTENT_VAR + _NEWRELIC_CALLBACK: Final = "newrelic" _NEWRELIC_VAR_PREFIX: Final = "newrelic_" _LANGFUSE_OTEL_CALLBACK: Final = "langfuse_otel" @@ -36,6 +38,9 @@ def callback_config_error( ) if langfuse_error is not None: return langfuse_error + capture_error: Final = _capture_message_content_error(callback_name, callback_vars, callback_type) + if capture_error is not None: + return capture_error if callback_name != _NEWRELIC_CALLBACK: return None return _newrelic_config_error(callback_vars) @@ -80,6 +85,32 @@ def _langfuse_span_scope_error(callback_name: str | None, callback_vars: Mapping return None +def _capture_message_content_error( + callback_name: str | None, callback_vars: Mapping[str, str], callback_type: str | None +) -> str | None: + value: Final = callback_vars.get(CAPTURE_MESSAGE_CONTENT_VAR) + if value is None: + return None + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.integrations.otel.presets.destinations import destination_capable_backends + from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + validate_capture_message_content_value, + ) + + supported: Final = sorted(destination_capable_backends()) + if callback_name not in supported: + return f"{CAPTURE_MESSAGE_CONTENT_VAR} applies to the OTel v2 callbacks {supported} only, not {callback_name!r}" + if not is_otel_v2_enabled(): + return f"Per-destination {CAPTURE_MESSAGE_CONTENT_VAR} requires the proxy to run with LITELLM_OTEL_V2=true." + try: + validate_capture_message_content_value(value) + except ValueError as e: + return str(e) + if callback_type == "failure": + return f"{CAPTURE_MESSAGE_CONTENT_VAR} needs callback_type 'success' or 'success_and_failure'" + return None + + # Which credential family a dynamic variable belongs to. The families are the # integrations that share one account: every langfuse_* variable configures the # same Langfuse project whether it rides the classic callback or the OTel one, @@ -190,20 +221,41 @@ def conflicting_span_scope_error( ) +def conflicting_capture_error( + callback_name: str, + callback_vars: Mapping[str, str] | None, + stored_entries: Sequence[tuple[str | None, Mapping[str, str]]], +) -> str | None: + incoming: Final = None if callback_vars is None else callback_vars.get(CAPTURE_MESSAGE_CONTENT_VAR) + if incoming is None: + return None + return next( + ( + f"{CAPTURE_MESSAGE_CONTENT_VAR} is already set to {stored!r} by another {callback_name} entry. " + f"Every {callback_name} entry shares one value: remove that entry or send the same value." + for name, entry_vars in stored_entries + if name == callback_name and (stored := entry_vars.get(CAPTURE_MESSAGE_CONTENT_VAR)) not in (None, incoming) + ), + None, + ) + + def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str | None: """Validate every ``logging`` entry of a team/key metadata payload.""" if not metadata: return None - entries: Final = metadata.get("logging") - if not isinstance(entries, Sequence) or isinstance(entries, (str, bytes)): + entries: Final = _object_sequence(metadata.get("logging")) + if entries is None: return None entry_vars: Final = tuple(_entry_callback_vars(entry) for entry in entries) + named_vars: Final = tuple(zip((_entry_callback_name(entry) for entry in entries), entry_vars)) return next( ( error for error in ( *(_logging_entry_error(entry) for entry in entries), *(conflicting_span_scope_error(entry_vars[i], entry_vars[:i]) for i in range(len(entry_vars))), + *(_conflicting_entry_capture_error(named_vars[i], named_vars[:i]) for i in range(len(named_vars))), ) if error is not None ), @@ -211,6 +263,31 @@ def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str ) +def stored_capture_entries(logging_entries: object) -> tuple[tuple[str | None, Mapping[str, str]], ...]: + """The callback name and vars of every stored ``logging`` entry, for ``conflicting_capture_error``.""" + entries: Final = _object_sequence(logging_entries) or () + return tuple((_entry_callback_name(entry), _entry_callback_vars(entry)) for entry in entries) + + +def _object_sequence(value: object) -> Sequence[object] | None: + return value if isinstance(value, Sequence) and not isinstance(value, (str, bytes)) else None + + +def _entry_callback_name(entry: object) -> str | None: + match entry: + case {"callback_name": str() as callback_name}: + return callback_name + case _: + return None + + +def _conflicting_entry_capture_error( + entry: tuple[str | None, Mapping[str, str]], earlier: Sequence[tuple[str | None, Mapping[str, str]]] +) -> str | None: + callback_name, callback_vars = entry + return None if callback_name is None else conflicting_capture_error(callback_name, callback_vars, earlier) + + def _entry_callback_vars(entry: object) -> Mapping[str, str]: callback_vars: Final = entry.get("callback_vars") if isinstance(entry, Mapping) else None if not isinstance(callback_vars, Mapping): diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 0d693b85ce8..fdc2e6d94b8 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -227,6 +227,7 @@ from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES from litellm.types.llms.anthropic import ANTHROPIC_API_HEADERS from litellm.types.services import ServiceTypes from litellm.types.utils import ( + CAPTURE_MESSAGE_CONTENT_VAR, CustomPricingLiteLLMParams, LlmProviders, ProviderSpecificHeader, @@ -975,6 +976,8 @@ def convert_key_logging_metadata_to_callback( team_callback_settings_obj.callbacks.append(data.callback_name) for var, value in data.callback_vars.items(): + if var == CAPTURE_MESSAGE_CONTENT_VAR: + continue # New Relic routing reads these from the trusted-vars overlay with no # callback-name check, so scope them to the newrelic entry: a team that # put newrelic_* under a different callback never asked for New Relic and @@ -1162,7 +1165,6 @@ def resolve_tenant_otel_destinations( span still routes to the tenant's credentials the way it did then. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled - from litellm.integrations.otel.presets.destinations import destination_for if not is_otel_v2_enabled(): return () @@ -1186,23 +1188,28 @@ def resolve_tenant_otel_destinations( for name in dict.fromkeys( callback.callback_name for callback in callbacks if callback.callback_type != "failure" ) - if ( - destination := destination_for( - name, - _tenant_otel_params( - MappingProxyType( - { - var: value - for callback in callbacks - if callback.callback_name == name - for var, value in callback.callback_vars.items() - } - ) - ), - _tenant_service_name(user_api_key_dict), + if (destination := _tenant_destination(name, callbacks, _tenant_service_name(user_api_key_dict))) is not None + ) + + +def _tenant_destination( + name: str, callbacks: Sequence[AddTeamCallback], service_name: str | None +) -> "OtelDestination | None": + from litellm.integrations.otel.model.config import parse_capture_message_content + from litellm.integrations.otel.presets.destinations import destination_for + + callback_vars: Final = MappingProxyType( + dict( + itertools.chain.from_iterable( + callback.callback_vars.items() for callback in callbacks if callback.callback_name == name ) ) - is not None + ) + return destination_for( + name, + _tenant_otel_params(callback_vars), + service_name, + parse_capture_message_content(callback_vars.get(CAPTURE_MESSAGE_CONTENT_VAR)), ) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 91fd447fb8d..3908ab3c927 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -31,8 +31,10 @@ from litellm.proxy._types import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.callback_config_validation import ( callback_config_error, + conflicting_capture_error, conflicting_span_scope_error, cross_entry_family_error, + stored_capture_entries, ) from litellm.proxy.common_utils.callback_utils import ( # noqa: F401 # legacy module exports _CALLBACK_VAR_ENCRYPTED_PREFIX, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -288,6 +290,9 @@ async def add_team_callbacks( - langfuse_host: The host for the Langfuse callback - langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT) - langfuse_span_scope: For langfuse_otel, "full" (default) sends the whole request trace, "llm_only" sends only the model-call spans + - capture_message_content: For OTel v2 callbacks, "no_content" or "span_only". + An explicit value overrides the global setting for this destination; omitted follows the proxy's own callback for that backend, else the global setting. + "span_only" puts prompt and response content on the destination's spans, "no_content" leaves it out - gcs_bucket_name: The name of the GCS bucket - gcs_path_service_account: The path to the GCS service account - langsmith_api_key: The API key for the Langsmith callback @@ -351,11 +356,19 @@ async def add_team_callbacks( # Decrypted, because the checks compare the incoming values against # the stored ones and the credentials are encrypted at rest. decrypted_logging: Final = decrypt_callback_vars(team_metadata).get("logging") + stored_capture: Final = stored_capture_entries(decrypted_logging) stored_entries: Final = decrypted_logging if isinstance(decrypted_logging, list) else () stored_entry_vars: Final = [entry.get("callback_vars") or {} for entry in stored_entries] scope_error: Final = conflicting_span_scope_error(data.callback_vars, stored_entry_vars) if scope_error is not None: raise _callback_config_error(scope_error) + capture_error: Final = conflicting_capture_error( + data.callback_name, + data.callback_vars, + stored_capture, + ) + if capture_error is not None: + raise _callback_config_error(capture_error) # One entry has to own a credential family end to end. The entries are # flattened into one dict before a request reads them, so an entry # naming only a destination would pair with a key written on another diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4a962e5b3bd..d6751610e7a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3726,6 +3726,28 @@ ArizeOtlpProtocol = Literal["grpc", "http/protobuf"] ARIZE_OTLP_PROTOCOLS: Final[frozenset[str]] = frozenset(get_args(ArizeOtlpProtocol)) +CAPTURE_MESSAGE_CONTENT_VAR: Final = "capture_message_content" + + +class CaptureMessageContent(str): + NO_CONTENT = "no_content" + SPAN_ONLY = "span_only" + EVENT_ONLY = "event_only" + SPAN_AND_EVENT = "span_and_event" + + +# The values a team or key destination may set. OTel v2 writes content only to span attributes, +# so the event modes stay global-only settings +CAPTURE_MESSAGE_CONTENT_VALUES: Final[frozenset[str]] = frozenset( + {CaptureMessageContent.NO_CONTENT, CaptureMessageContent.SPAN_ONLY} +) + + +def captures_span_content(mode: str | None) -> bool: + """Whether a capture mode puts prompt and response content on spans""" + return mode in (CaptureMessageContent.SPAN_ONLY, CaptureMessageContent.SPAN_AND_EVENT) + + class StandardCallbackDynamicParams(TypedDict, total=False): # Langfuse dynamic params langfuse_public_key: str | None diff --git a/tests/integration/_support/otlp_sink.py b/tests/integration/_support/otlp_sink.py index eeabe9d886f..a4d2ea30959 100644 --- a/tests/integration/_support/otlp_sink.py +++ b/tests/integration/_support/otlp_sink.py @@ -186,6 +186,7 @@ class _Handler(BaseHTTPRequestHandler): "count": len(recorded), "host": self.headers.get("host", ""), "headers": dict(self.headers), + "span_ids": [span["span_id"] for span in recorded], } ) self._send_json({"recorded": len(recorded)}, status=self.state.status) @@ -466,9 +467,7 @@ def _await_sink(url: str) -> None: def owned_grpc_sink(directory: Path) -> Iterator[GrpcSink]: http_port: Final = _free_port() grpc_port: Final = _free_port() - with _spawn_sink( - directory, "otlp-grpc-sink.log", ["--port", str(http_port), "--grpc-port", str(grpc_port)] - ): + with _spawn_sink(directory, "otlp-grpc-sink.log", ["--port", str(http_port), "--grpc-port", str(grpc_port)]): control_url: Final = f"http://127.0.0.1:{http_port}" _await_sink(control_url) yield GrpcSink(url=f"http://127.0.0.1:{grpc_port}", control_url=control_url) diff --git a/tests/integration/observability/test_otel_team_capture_message_content.py b/tests/integration/observability/test_otel_team_capture_message_content.py new file mode 100644 index 00000000000..84da9a40486 --- /dev/null +++ b/tests/integration/observability/test_otel_team_capture_message_content.py @@ -0,0 +1,720 @@ +import json +import re +import uuid +from collections.abc import Generator, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal +from urllib.parse import urlparse + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment, object_value +from integration._support.otlp_sink import ( + ConnectSink, + Span, + SpanSinks, + configure_sink, + owned_connect_sink, + recorded_requests, + recorded_spans, +) +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +MARKER: Final = re.compile(rb"cmc-(?:tool-|fail-)?[0-9a-f]{32}") +JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +SERVER: Final = 2 +CALL_ID: Final = "litellm.call_id" +OPERATION: Final = "gen_ai.operation.name" +INPUT_TOKENS: Final = "gen_ai.usage.input_tokens" +Endpoint = Literal["chat", "responses", "messages"] +Capture = Literal["no_content", "span_only"] + + +def _marker(kind: str = "") -> str: + return f"cmc-{kind}{uuid.uuid4().hex}" + + +def _usage() -> dict[str, JsonValue]: + return {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9} + + +def _chat_reply(marker: str, stream: bool) -> Reply: + identity: Final = f"chatcmpl-{uuid.uuid4().hex}" + if marker.startswith("cmc-tool-"): + call: Final[dict[str, JsonValue]] = { + "id": "call_1", + "type": "function", + "function": {"name": "lookup", "arguments": json.dumps({"query": marker})}, + } + message: Final[dict[str, JsonValue]] = {"role": "assistant", "content": None, "tool_calls": [call]} + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": message, "finish_reason": "tool_calls"}], + "usage": _usage(), + } + ).encode() + ) + if not stream: + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": marker}, "finish_reason": "stop"} + ], + "usage": _usage(), + } + ).encode() + ) + chunk: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini"} + deltas: Final[tuple[dict[str, JsonValue], ...]] = ( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": marker}}]}, + {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**chunk, "choices": [], "usage": _usage()}, + ) + return Reply( + content_type="text/event-stream", + chunks=(*(b"data: " + json.dumps(delta).encode() + b"\n\n" for delta in deltas), b"data: [DONE]\n\n"), + ) + + +def _responses_reply(marker: str, stream: bool) -> Reply: + identity: Final = uuid.uuid4().hex + response: Final[dict[str, JsonValue]] = { + "id": f"resp_{identity}", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "id": f"msg_{identity}", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": marker, "annotations": []}], + } + ], + "usage": {"input_tokens": 7, "output_tokens": 2, "total_tokens": 9}, + } + if not stream: + return Reply(body=json.dumps(response).encode()) + events: Final[tuple[dict[str, JsonValue], ...]] = ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**response, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": f"msg_{identity}", + "output_index": 0, + "content_index": 0, + "delta": marker, + }, + {"type": "response.completed", "sequence_number": 2, "response": response}, + ) + return Reply( + content_type="text/event-stream", + chunks=tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events), + ) + + +def _upstream(request: Request) -> Reply: + found: Final = MARKER.search(request.body) + if found is None: + return Reply(status=404, body=b'{"error":"no marker"}') + marker: Final = found.group(0).decode() + if marker.startswith("cmc-fail-"): + return Reply(status=500, body=b'{"error":{"message":"scripted upstream failure","type":"server_error"}}') + stream: Final = object_value(JSON.validate_json(request.body)).get("stream") is True + if request.target.endswith("/responses"): + return _responses_reply(marker, stream) + return _chat_reply(marker, stream) + + +def _body(model: str, endpoint: Endpoint, marker: str, stream: bool) -> tuple[str, dict[str, JsonValue]]: + if endpoint == "responses": + return "/v1/responses", {"model": model, "input": marker, "stream": stream} + messages: Final[list[JsonValue]] = [{"role": "user", "content": marker}] + if endpoint == "messages": + return "/v1/messages", {"model": model, "max_tokens": 16, "messages": messages, "stream": stream} + return "/v1/chat/completions", {"model": model, "messages": messages, "stream": stream} + + +@dataclass(frozen=True, slots=True) +class Sent: + marker: str + trace_id: str + call_id: str + status: int + text: str + + +@dataclass(frozen=True, slots=True) +class Cursors: + operator: int + tenant: int + arize: int + newrelic: int + + +@dataclass(frozen=True, slots=True) +class Rig: + proxy: Gateway + owned: OwnedProxy + scenario: Scenario + model: str + upstream: Wire + sinks: SpanSinks + newrelic: ConnectSink + + def cursors(self) -> Cursors: + self.upstream.drain() + return Cursors( + recorded_spans(self.sinks.operator)[0], + recorded_spans(self.sinks.tenant)[0], + recorded_spans(self.sinks.arize)[0], + recorded_spans(self.newrelic.control_url)[0], + ) + + def upstream_hits(self, marker: str) -> int: + return sum(1 for request in self.upstream.drain() if marker.encode() in request.body) + + def langfuse(self, name: str, host: str | None = None) -> dict[str, str]: + return { + "langfuse_public_key": f"pk-lf-{name}", + "langfuse_secret_key": f"sk-lf-{name}", + "langfuse_host": host or self.sinks.tenant, + } + + def attach(self, team: str, callback: str, variables: Mapping[str, str], kind: str | None = None) -> httpx.Response: + body: Final[dict[str, JsonValue]] = {"callback_name": callback, "callback_vars": dict(variables)} + return self.proxy.request( + "POST", f"/team/{team}/callback", body if kind is None else {**body, "callback_type": kind} + ) + + def team_key( + self, + capture: Capture | None, + name: str, + *, + callback: str = "langfuse_otel", + variables: Mapping[str, str] | None = None, + kind: str | None = None, + ) -> tuple[str, str]: + team: Final = self.scenario.team() + base: Final = dict(variables) if variables is not None else self.langfuse(name) + attached: Final = self.attach( + team, callback, base if capture is None else {**base, "capture_message_content": capture}, kind + ) + assert attached.status_code == 200, attached.text + return team, self.scenario.key(team_id=team) + + def send( + self, + key: str, + endpoint: Endpoint = "chat", + stream: bool = False, + *, + marker: str | None = None, + extra: Mapping[str, JsonValue] | None = None, + ) -> Sent: + tagged: Final = marker or _marker() + path, body = _body(self.model, endpoint, tagged, stream) + trace_id: Final = uuid.uuid4().hex + call_id: Final = str(uuid.uuid4()) + headers: Final = { + "Authorization": f"Bearer {key}", + "traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01", + "x-litellm-call-id": call_id, + } + with self.proxy.client.stream("POST", path, json={**body, **(extra or {})}, headers=headers) as response: + text: Final = response.read().decode() + return Sent(tagged, trace_id, call_id, response.status_code, text) + + +def _served(sent: Sent) -> Sent: + assert sent.status == 200, sent.text + assert sent.marker in sent.text, sent.text + return sent + + +Credential = tuple[str, str] + + +def _sent_with(url: str, credential: Credential) -> frozenset[str]: + """Span ids ``url`` received in export requests whose ``credential`` header matched.""" + header, value = credential + return frozenset( + span_id + for request in recorded_requests(url) + if {name.lower(): seen for name, seen in object_value(request.get("headers") or {}).items()}.get(header) + == value + for span_id in request.get("span_ids") or () + if isinstance(span_id, str) + ) + + +def _of_request(url: str, since: int, sent: Sent, credential: Credential | None = None) -> tuple[Span, ...]: + """Spans of ``sent``'s trace at ``url``, narrowed to one account's export requests when ``credential`` is set.""" + spans: Final = recorded_spans(url, since)[1] + traces: Final = {sent.trace_id} | { + span["trace_id"] for span in spans if span["attributes"].get(CALL_ID) == sent.call_id + } + owned: Final = None if credential is None else _sent_with(url, credential) + return tuple(span for span in spans if span["trace_id"] in traces and (owned is None or span["span_id"] in owned)) + + +def _carries(span: Span, marker: str) -> bool: + return marker in json.dumps(span["attributes"]) + + +def _model_spans(spans: tuple[Span, ...]) -> tuple[Span, ...]: + return tuple(span for span in spans if OPERATION in span["attributes"]) + + +def _names(spans: tuple[Span, ...]) -> list[str]: + return sorted(span["name"] for span in spans) + + +def _with_content(url: str, since: int, sent: Sent) -> tuple[Span, ...]: + """The copy of ``sent`` at ``url`` once its model span arrived and some span carries the marker.""" + return eventually( + lambda: _of_request(url, since, sent), + lambda spans: bool(_model_spans(spans)) and any(_carries(span, sent.marker) for span in spans), + seconds=40, + ) + + +def _arrived( + url: str, since: int, sent: Sent, expected: frozenset[str], credential: Credential | None = None +) -> tuple[Span, ...]: + """The copy of ``sent`` at ``url`` once every span name in ``expected`` and a usage-bearing model span landed.""" + spans: Final = eventually( + lambda: _of_request(url, since, sent, credential), + lambda found: ( + expected <= {span["name"] for span in found} + and any(INPUT_TOKENS in span["attributes"] for span in _model_spans(found)) + ), + seconds=40, + ) + assert all( + span["attributes"].get(INPUT_TOKENS) == 7 for span in _model_spans(spans) if INPUT_TOKENS in span["attributes"] + ) + return spans + + +def _assert_redacted_twin( + rig: Rig, sent: Sent, cursors: Cursors, sink: str, since: int, credential: Credential | None = None +) -> tuple[Span, ...]: + operator: Final = _with_content(rig.sinks.operator, cursors.operator, sent) + carried: Final = frozenset(span["name"] for span in operator if _carries(span, sent.marker)) + tenant: Final = _arrived(sink, since, sent, carried, credential) + leaked: Final = [span["name"] for span in tenant if _carries(span, sent.marker)] + assert leaked == [], f"{leaked} carried content; tenant saw {_names(tenant)}" + model: Final = _model_spans(tenant)[0]["attributes"] + assert "gen_ai.input.messages" not in model and "gen_ai.output.messages" not in model, sorted(model) + assert model.get("gen_ai.request.model"), sorted(model) + return tenant + + +def _assert_content_twin(rig: Rig, sent: Sent, cursors: Cursors) -> None: + operator: Final = _with_content(rig.sinks.operator, cursors.operator, sent) + carried: Final = frozenset(span["name"] for span in operator if _carries(span, sent.marker)) + tenant: Final = _arrived(rig.sinks.tenant, cursors.tenant, sent, carried) + kept: Final = frozenset(span["name"] for span in tenant if _carries(span, sent.marker)) + assert carried <= kept, f"content lost on {sorted(carried - kept)}" + + +@pytest.mark.parametrize( + ("endpoint", "stream"), + [("chat", False), ("chat", True), ("responses", False), ("messages", True)], + ids=["chat-unary", "chat-stream", "responses-unary", "messages-stream"], +) +def test_no_content_team_keeps_its_trace_and_metadata_without_content( + rig: Rig, endpoint: Endpoint, stream: bool +) -> None: + _, key = rig.team_key("no_content", "a") + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key, endpoint, stream)) + assert rig.upstream_hits(sent.marker) == 1 + _assert_redacted_twin(rig, sent, cursors, rig.sinks.tenant, cursors.tenant) + + +@pytest.mark.parametrize("capture", ["span_only", None], ids=["span_only", "omitted"]) +def test_span_only_and_omitted_teams_keep_the_globally_captured_content(rig: Rig, capture: Capture | None) -> None: + _, key = rig.team_key(capture, "b") + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + _assert_content_twin(rig, sent, cursors) + + +def test_tool_call_arguments_never_reach_a_no_content_team(rig: Rig) -> None: + _, key = rig.team_key("no_content", "a") + cursors: Final = rig.cursors() + tool: Final[dict[str, JsonValue]] = { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {"query": {"type": "string"}}}}, + } + sent: Final = rig.send(key, marker=_marker("tool-"), extra={"tools": [tool]}) + assert sent.status == 200 and sent.marker in sent.text, sent.text + operator: Final = _with_content(rig.sinks.operator, cursors.operator, sent) + assert any(sent.marker in str(span["attributes"].get("gen_ai.output.messages", "")) for span in operator), ( + "operator copy lacks the tool-call arguments" + ) + tenant: Final = _assert_redacted_twin(rig, sent, cursors, rig.sinks.tenant, cursors.tenant) + assert "lookup" in json.dumps([span["attributes"] for span in tenant]), "tool definitions are metadata and stay" + + +def test_cache_hit_twin_of_a_no_content_team_is_redacted(rig: Rig) -> None: + _, key = rig.team_key("no_content", "a") + marker: Final = _marker() + _served(rig.send(key, marker=marker)) + assert rig.upstream_hits(marker) == 1 + cursors: Final = rig.cursors() + hit: Final = eventually( + lambda: _served(rig.send(key, marker=marker)), lambda _: rig.upstream_hits(marker) == 0, seconds=20 + ) + operator: Final = eventually( + lambda: _of_request(rig.sinks.operator, cursors.operator, hit), + lambda spans: any(span["kind"] == SERVER for span in spans), + seconds=40, + ) + expected: Final = frozenset(span["name"] for span in operator if span["kind"] == SERVER) + tenant: Final = eventually( + lambda: _of_request(rig.sinks.tenant, cursors.tenant, hit), + lambda spans: expected <= {span["name"] for span in spans}, + seconds=40, + ) + assert [span["name"] for span in tenant if _carries(span, marker)] == [], _names(tenant) + + +def test_a_failure_only_entry_rejects_the_setting(rig: Rig) -> None: + team: Final = rig.scenario.team() + response: Final = rig.attach( + team, "langfuse_otel", {**rig.langfuse("fail"), "capture_message_content": "no_content"}, "failure" + ) + assert response.status_code == 400, response.text + assert "success_and_failure" in response.text, response.text + + +def test_key_level_logging_with_no_content_is_redacted(rig: Rig) -> None: + logging: Final[list[JsonValue]] = [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {**rig.langfuse("key"), "capture_message_content": "no_content"}, + } + ] + key: Final = rig.scenario.key(metadata={"logging": logging}) + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + _assert_redacted_twin(rig, sent, cursors, rig.sinks.tenant, cursors.tenant) + + +def test_same_account_destination_redacts_the_shared_copy_only_for_that_team(rig: Rig) -> None: + _, restricted = rig.team_key("no_content", "operator", variables=rig.langfuse("operator", host=rig.sinks.operator)) + _, other = rig.team_key("span_only", "b") + cursors: Final = rig.cursors() + shared: Final = _served(rig.send(restricted)) + assert rig.upstream_hits(shared.marker) == 1 + copy: Final = eventually( + lambda: _of_request(rig.sinks.operator, cursors.operator, shared), + lambda spans: any(INPUT_TOKENS in span["attributes"] for span in _model_spans(spans)), + seconds=40, + ) + assert [span["name"] for span in copy if _carries(span, shared.marker)] == [], _names(copy) + assert len([span for span in copy if span["kind"] == SERVER and span["name"].startswith("POST ")]) == 1, _names( + copy + ) + _assert_content_twin(rig, _served(rig.send(other)), cursors) + + +def test_arize_destination_drops_openinference_content(rig: Rig) -> None: + _, key = rig.team_key( + "no_content", "arize", callback="arize", variables={"arize_space_key": "space-a", "arize_api_key": "arize-a"} + ) + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + operator: Final = _with_content(rig.sinks.operator, cursors.operator, sent) + assert any(sent.marker in str(span["attributes"].get("input.value", "")) for span in operator), ( + "operator copy has no OpenInference content to strip" + ) + tenant: Final = _assert_redacted_twin( + rig, sent, cursors, rig.sinks.arize, cursors.arize, ("arize-space-id", "space-a") + ) + model: Final = _model_spans(tenant)[0]["attributes"] + assert not any(name.startswith(("llm.input_messages.", "llm.output_messages.")) for name in model), sorted(model) + assert "input.value" not in model and "output.value" not in model, sorted(model) + assert model.get("llm.model_name"), sorted(model) + + +def test_weave_destination_drops_its_content(rig: Rig) -> None: + _, key = rig.team_key( + "no_content", + "weave", + callback="weave_otel", + variables={"wandb_api_key": "wandb-a", "weave_project_id": "team/a"}, + ) + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + operator: Final = eventually( + lambda: _of_request(rig.sinks.arize, cursors.arize, sent, ("project_id", "operator/weave")), + lambda spans: any(sent.marker in str(span["attributes"].get("weave.output", "")) for span in spans), + seconds=40, + ) + assert operator, "operator weave copy has no weave.output to strip" + tenant: Final = _assert_redacted_twin(rig, sent, cursors, rig.sinks.arize, cursors.arize, ("project_id", "team/a")) + assert not any("weave.output" in span["attributes"] for span in tenant), _names(tenant) + + +def test_newrelic_destination_drops_genai_content(rig: Rig) -> None: + _, key = rig.team_key("no_content", "newrelic", callback="newrelic", variables={"newrelic_api_key": "nr-team-a"}) + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + tenant: Final = _assert_redacted_twin( + rig, sent, cursors, rig.newrelic.control_url, cursors.newrelic, ("api-key", "nr-team-a") + ) + hosts: Final = { + object_value(request.get("headers") or {}).get("Host") + for request in recorded_requests(rig.newrelic.control_url) + if object_value(request.get("headers") or {}).get("api-key") == "nr-team-a" + } + assert hosts == {"otlp.nr-data.net"}, hosts + assert tenant + + +def test_span_only_team_lifts_global_no_content_only_for_its_destination(dark_rig: Rig) -> None: + """The sibling Arize entry gets a fan-out destination and the SigNoz entry rides the routed tracer.""" + team, key = dark_rig.team_key("span_only", "b") + siblings: Final = ( + dark_rig.attach(team, "arize", {"arize_space_key": "space-sibling", "arize_api_key": "arize-sibling"}), + dark_rig.attach( + team, + "signoz", + {"signoz_ingestion_endpoint": dark_rig.sinks.arize + "/v1/traces", "signoz_ingestion_key": "signoz-team"}, + ), + ) + assert [response.status_code for response in siblings] == [200, 200], [response.text for response in siblings] + cursors: Final = dark_rig.cursors() + sent: Final = _served(dark_rig.send(key)) + assert dark_rig.upstream_hits(sent.marker) == 1 + operator: Final = _arrived(dark_rig.sinks.operator, cursors.operator, sent, frozenset()) + tenant: Final = _with_content(dark_rig.sinks.tenant, cursors.tenant, sent) + sibling: Final = _arrived( + dark_rig.sinks.arize, + cursors.arize, + sent, + frozenset(), + ("arize-space-id", "space-sibling"), + ) + signoz: Final = _arrived( + dark_rig.sinks.arize, + cursors.arize, + sent, + frozenset(), + ("signoz-ingestion-key", "signoz-team"), + ) + assert any(_carries(span, sent.marker) for span in tenant), _names(tenant) + assert not any(_carries(span, sent.marker) for span in operator), _names(operator) + assert not any(_carries(span, sent.marker) for span in sibling), _names(sibling) + assert not any(_carries(span, sent.marker) for span in signoz), _names(signoz) + + +def test_an_unsupported_value_fails_registration(rig: Rig) -> None: + team: Final = rig.scenario.team() + response: Final = rig.attach(team, "langfuse_otel", {**rig.langfuse("a"), "capture_message_content": "bogus"}) + assert response.status_code == 422, response.text + assert "Invalid capture_message_content 'bogus'" in response.text, response.text + + +def test_classic_langfuse_rejects_the_setting(rig: Rig) -> None: + team: Final = rig.scenario.team() + response: Final = rig.attach(team, "langfuse", {**rig.langfuse("a"), "capture_message_content": "no_content"}) + assert response.status_code == 400, response.text + assert "capture_message_content" in response.text, response.text + + +def test_a_second_entry_for_the_same_backend_cannot_disagree(rig: Rig) -> None: + team, _ = rig.team_key("no_content", "a") + response: Final = rig.attach(team, "langfuse_otel", {"capture_message_content": "span_only"}, "success_and_failure") + assert response.status_code == 400, response.text + assert "already set to 'no_content' by another langfuse_otel entry" in response.text, response.text + + +def test_replacing_the_registration_switches_the_team_to_content(rig: Rig) -> None: + team, key = rig.team_key("no_content", "a") + cursors: Final = rig.cursors() + _assert_redacted_twin(rig, _served(rig.send(key)), cursors, rig.sinks.tenant, cursors.tenant) + removed: Final = rig.proxy.request("DELETE", f"/team/{team}/callback/langfuse_otel") + assert removed.status_code == 200, removed.text + replaced: Final = rig.attach(team, "langfuse_otel", {**rig.langfuse("a"), "capture_message_content": "span_only"}) + assert replaced.status_code == 200, replaced.text + + def attempt() -> bool: + since: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + operator: Final = _with_content(rig.sinks.operator, since.operator, sent) + carried: Final = frozenset(span["name"] for span in operator if _carries(span, sent.marker)) + tenant: Final = _arrived(rig.sinks.tenant, since.tenant, sent, carried) + return any(_carries(span, sent.marker) for span in tenant) + + assert eventually(attempt, bool, seconds=60) + + +def test_request_metadata_cannot_lift_the_team_restriction(rig: Rig) -> None: + _, key = rig.team_key("no_content", "a") + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key, extra={"metadata": {"capture_message_content": "span_only"}})) + assert rig.upstream_hits(sent.marker) == 1 + _assert_redacted_twin(rig, sent, cursors, rig.sinks.tenant, cursors.tenant) + + +def test_stalled_tenant_during_a_mixed_burst_keeps_serving_and_each_teams_policy(rig: Rig) -> None: + _, restricted = rig.team_key("no_content", "a") + _, open_key = rig.team_key("span_only", "b") + shapes: Final[tuple[tuple[Endpoint, bool], ...]] = ( + ("chat", False), + ("chat", True), + ("responses", False), + ("messages", True), + ) + plan: Final = [(restricted if index % 2 else open_key, *shapes[index % len(shapes)]) for index in range(30)] + cursors: Final = rig.cursors() + configure_sink(rig.sinks.tenant, paused=True) + try: + with ThreadPoolExecutor(max_workers=10) as pool: + sent: Final = list(pool.map(lambda job: rig.send(job[0], job[1], job[2]), plan)) + finally: + configure_sink(rig.sinks.tenant, paused=False) + assert [result.status for result in sent] == [200] * 30, [result.text for result in sent if result.status != 200] + for (key, _, _), result in zip(plan, sent, strict=True): + roots = [ + span + for span in _with_content(rig.sinks.operator, cursors.operator, result) + if span["kind"] == SERVER and span["name"].startswith("POST ") + ] + assert len(roots) == 1, roots + if key == restricted: + _assert_redacted_twin(rig, result, cursors, rig.sinks.tenant, cursors.tenant) + else: + _assert_content_twin(rig, result, cursors) + + +def _write_config(directory: Path, sinks: SpanSinks, name: str, callbacks: tuple[str, ...]) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["litellm_settings"] = { + **config.get("litellm_settings", {}), + "callbacks": list(callbacks), + "otel_tenant_destination_mode": "additive", + "provider_url_destination_allowed_hosts": [ + urlparse(sinks.tenant).netloc, + urlparse(sinks.operator).netloc, + urlparse(sinks.arize).netloc, + ], + } + config["general_settings"] = {**config.get("general_settings", {}), "user_api_key_cache_ttl": 2} + path: Final = directory / f"{name}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@contextmanager +def _started( + provider: Wire, + sinks: SpanSinks, + newrelic: ConnectSink, + directory: Path, + capture: str | None, + callbacks: tuple[str, ...] = ("langfuse_otel",), +) -> Generator[Rig]: + environment: Final = { + "LITELLM_OTEL_V2": "1", + "OTEL_BSP_SCHEDULE_DELAY": "300", + "MAPPER_NAMES": "genai,openinference", + "LANGFUSE_HOST": sinks.operator, + "LANGFUSE_PUBLIC_KEY": "pk-lf-operator", + "LANGFUSE_SECRET_KEY": "sk-lf-operator", + "ARIZE_SPACE_KEY": "space-operator", + "ARIZE_API_KEY": "arize-operator", + "ARIZE_HTTP_ENDPOINT": sinks.arize + "/v1/traces", + "WANDB_API_KEY": "wandb-operator", + "WANDB_PROJECT_ID": "operator/weave", + "WANDB_HOST": sinks.arize, + "SIGNOZ_INGESTION_ENDPOINT": sinks.arize + "/v1/traces", + "SIGNOZ_INGESTION_KEY": "signoz-operator", + "HTTPS_PROXY": newrelic.proxy_url, + "NO_PROXY": "127.0.0.1,localhost", + "REQUESTS_CA_BUNDLE": newrelic.ca_pem, + **({} if capture is None else {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": capture}), + } + with ( + gateway_from_environment() as gateway, + owned_proxy_process( + gateway, + directory, + environment, + config=_write_config(directory, sinks, "capture", callbacks), + remove_environment=( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", + "LITELLM_OTEL_TENANT_DESTINATION_MODE", + ), + workers=2, + ) as owned, + owned.gateway.scenario() as scenario, + ): + yield Rig( + owned.gateway, owned, scenario, scenario.model(api_base=provider.url + "/v1"), provider, sinks, newrelic + ) + + +@pytest.fixture(scope="module") +def provider() -> Iterator[Wire]: + with wire_server(_upstream) as wire: + yield wire + + +@pytest.fixture(scope="module") +def newrelic_sink(tmp_path_factory: pytest.TempPathFactory) -> Iterator[ConnectSink]: + with owned_connect_sink(tmp_path_factory.mktemp("newrelic-sink")) as sink: + yield sink + + +@pytest.fixture(scope="module") +def rig( + provider: Wire, audit_sinks: SpanSinks, newrelic_sink: ConnectSink, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("capture-span-only") + with _started(provider, audit_sinks, newrelic_sink, directory, "span_only") as started: + yield started + + +@pytest.fixture(scope="module") +def dark_rig( + provider: Wire, audit_sinks: SpanSinks, newrelic_sink: ConnectSink, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[Rig]: + with _started( + provider, + audit_sinks, + newrelic_sink, + tmp_path_factory.mktemp("capture-no-content"), + None, + ("langfuse_otel", "signoz"), + ) as started: + yield started diff --git a/tests/unit/integrations/otel/test_langfuse_logger.py b/tests/unit/integrations/otel/test_langfuse_logger.py index aca9dcc8a5e..ff2060d91c2 100644 --- a/tests/unit/integrations/otel/test_langfuse_logger.py +++ b/tests/unit/integrations/otel/test_langfuse_logger.py @@ -16,6 +16,7 @@ import litellm # noqa: E402 from litellm.caching.dual_cache import DualCache # noqa: E402 from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger # noqa: E402 from litellm.integrations.otel.model.config import OpenTelemetryV2Config, is_otel_v2_enabled # noqa: E402 +from litellm.integrations.otel.model.destination import OtelDestination # noqa: E402 from litellm.integrations.otel.model.spans import LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole # noqa: E402 from litellm.integrations.otel.plumbing import context as otel_context # noqa: E402 from litellm.integrations.otel.plumbing import providers # noqa: E402 @@ -155,6 +156,25 @@ def test_chat_request_stamps_root_observation_input_and_output(): assert [(turn["role"], turn["content"]) for turn in output] == [("assistant", "pong")] +def test_content_on_destination_stamps_langfuse_root_io_when_global_capture_is_off() -> None: + logger, exporter = _logger(capture="no_content") + destination: Final = OtelDestination( + endpoint="https://team.example.com/api/public/otel", + callback_name="langfuse_otel", + capture_message_content="span_only", + ) + token: Final = otel_context.set_request_destinations((destination,)) + try: + response: Final = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + _run_request(logger, CHAT_DATA, "acompletion", response) + finally: + otel_context.reset_request_destinations(token) + + attrs: Final = _root_attrs(exporter) + assert json.loads(attrs[INPUT_ATTR]) == [{"role": "user", "content": "ping"}] + assert json.loads(attrs[OUTPUT_ATTR]) == [{"role": "assistant", "content": "pong"}] + + def test_responses_request_folds_instructions_into_input_and_stamps_output_items(): logger, exporter = _logger() data = {"model": "gpt-5.4-mini", "instructions": "be terse", "input": "ping"} @@ -461,7 +481,9 @@ def test_a_request_without_trace_controls_stamps_none_of_them(): logger, exporter = _logger() root_attrs, generation_attrs = _run_named_request( - logger, exporter, {"metadata": {"user_api_key_team_id": "t1", "tags": []}, "proxy_server_request": {"headers": {}}} + logger, + exporter, + {"metadata": {"user_api_key_team_id": "t1", "tags": []}, "proxy_server_request": {"headers": {}}}, ) assert set(TRACE_CONTROL_ATTRS).isdisjoint(root_attrs) @@ -472,7 +494,7 @@ def test_a_request_without_trace_controls_stamps_none_of_them(): ("capture", "mappers"), [("no_content", ("genai", "langfuse")), ("span_only", ("genai",))], ) -def test_factory_keeps_the_base_logger_unless_langfuse_content_capture_is_on(capture, mappers): +def test_langfuse_root_io_requires_global_or_destination_capture(capture: str, mappers: tuple[str, ...]) -> None: logger, exporter = _logger(capture=capture, mappers=mappers) _run_request(logger, CHAT_DATA, "acompletion", ModelResponse()) @@ -484,13 +506,13 @@ def test_factory_keeps_the_base_logger_unless_langfuse_content_capture_is_on(cap ("capture", "mappers", "relays_streams"), [ ("span_only", ("genai", "langfuse"), True), - ("no_content", ("genai", "langfuse"), False), + ("no_content", ("genai", "langfuse"), True), ("span_only", ("genai",), False), ], ) -def test_only_langfuse_content_capture_takes_proxy_streams_off_the_fast_path( - monkeypatch, capture, mappers, relays_streams -): +def test_langfuse_mapper_relays_streams_for_per_request_content_capture( + monkeypatch: pytest.MonkeyPatch, capture: str, mappers: tuple[str, ...], relays_streams: bool +) -> None: logger, _ = _logger(capture=capture, mappers=mappers) monkeypatch.setattr(litellm, "callbacks", [logger]) diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index 2d7e0745039..5c9cab88784 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -5,6 +5,7 @@ import contextvars import time from base64 import b64encode from collections.abc import Mapping +from dataclasses import replace from datetime import datetime, timezone from functools import reduce from types import MappingProxyType @@ -13,13 +14,14 @@ from typing import Final import pytest from opentelemetry import trace as trace_api from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.trace import Status, StatusCode import litellm from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.otel import GenAIOperation from litellm.integrations.otel import logger as otel_logger from litellm.integrations.otel.logger import ( OpenTelemetryV2, @@ -27,13 +29,24 @@ from litellm.integrations.otel.logger import ( fan_out_provider, publish_global_otel_v2_provider, ) +from litellm.integrations.otel.mappers import resolve_mappers from litellm.integrations.otel.model.config import ( + CaptureMessageContent, ExporterOwner, ExporterSpec, OpenTelemetryV2Config, is_otel_v2_enabled, ) from litellm.integrations.otel.model.destination import OtelDestination +from litellm.integrations.otel.model.payloads import ( + LLMCallSpanData, + LLMRequestParams, + LLMUsage, + MCPToolCallSpanData, + RequestIdentity, + ServerInfo, + ToolDefinition, +) from litellm.integrations.otel.plumbing import providers as otel_providers from litellm.integrations.otel.plumbing.context import ( destination_backends, @@ -44,9 +57,11 @@ from litellm.integrations.otel.plumbing.providers import ( TenantFanOutSpanProcessor, _OverriddenBackendFilter, _sink_key, + attach_tenant_fan_out, build_tracer_provider, deliverable_destinations, operator_sink_scopes, + register_exporter_factory, ) from litellm.integrations.otel.plumbing.routing import TenantTracerCache, get_tracer from litellm.integrations.otel.presets.arize import arize_preset @@ -55,7 +70,7 @@ from litellm.integrations.otel.presets.destinations import ( destination_for, ) from litellm.integrations.otel.presets.langfuse import langfuse_preset -from litellm.proxy._types import AddTeamCallback, UserAPIKeyAuth +from litellm.proxy._types import AddTeamCallback, TeamCallbackMetadata, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import ( convert_key_logging_metadata_to_callback, resolve_tenant_otel_destinations, @@ -4228,6 +4243,569 @@ class TestTenantHostSsrfGuard: assert sum("provider_url_destination_allowed_hosts" in record.message for record in caplog.records) == 1 +_ALL_MAPPERS: Final = ("genai", "legacy", "openinference", "langfuse", "weave", "langtrace") +_SECRET: Final = "SECRET-MARKER" +_MODEL_CALL_WITH_CONTENT: Final = LLMCallSpanData( + operation=GenAIOperation.CHAT, + provider="openai", + request_model="gpt-4o", + response_model="gpt-4o-2024", + response_id="resp_1", + request_params=LLMRequestParams(temperature=0.5, max_tokens=128), + usage=LLMUsage(input_tokens=12, output_tokens=8, total_tokens=20), + finish_reasons=("tool_calls",), + error=None, + response_cost=0.001, + server=ServerInfo("api.openai.com", 443), + identity=RequestIdentity(call_id="c1", team_id="t1"), + tools=(ToolDefinition(name="lookup", description="Find a city", parameters_json='{"type":"object"}'),), + messages_in=( + {"role": "system", "content": f"system {_SECRET}"}, + {"role": "user", "content": f"prompt {_SECRET}"}, + ), + choices_out=( + { + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": f"answer {_SECRET}", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "lookup", "arguments": f'{{"city": "{_SECRET}"}}'}, + } + ], + }, + }, + ), +) +_MODEL_CALL_WITHOUT_CONTENT: Final = replace(_MODEL_CALL_WITH_CONTENT, messages_in=(), choices_out=()) +_TOOL_CALL_WITH_CONTENT: Final = MCPToolCallSpanData( + operation=GenAIOperation.EXECUTE_TOOL, + method="tools/call", + tool_name="lookup", + server_name="maps", + server_address="maps.local", + server_port=443, + session_id="s1", + arguments_json=f'{{"city": "{_SECRET}"}}', + result_json=f'{{"weather": "{_SECRET}"}}', + error=None, + response_cost=None, + identity=RequestIdentity(call_id="c2", team_id="t1"), +) +_TOOL_CALL_WITHOUT_CONTENT: Final = replace(_TOOL_CALL_WITH_CONTENT, arguments_json=None, result_json=None) + +NO_CONTENT_DEST = OtelDestination( + endpoint="http://team-a.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic YQ=="}), + callback_name="langfuse_otel", + capture_message_content="no_content", +) +INHERITING_DEST = OtelDestination( + endpoint="http://team-b.local/v1", + headers=MappingProxyType({"api_key": "k", "arize-space-id": "s"}), + callback_name="arize", +) + + +def mapped(data: LLMCallSpanData | MCPToolCallSpanData) -> Mapping[str, object]: + """What the configured mappers write on the span, the way the logger stamps it.""" + return MappingProxyType(reduce(lambda acc, mapper: {**acc, **mapper.map(data)}, resolve_mappers(_ALL_MAPPERS), {})) + + +def recorded(attributes: Mapping[str, object]) -> dict[str, object]: + """``attributes`` the way the SDK records them, sequences frozen into tuples.""" + return {key: tuple(value) if isinstance(value, list) else value for key, value in attributes.items()} + + +def model_call_tree(provider: TracerProvider, attributes: Mapping[str, object]) -> None: + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("chat gpt-4o") as llm: + llm.set_attributes(attributes) + llm.add_event("gen_ai.content.first_chunk", {"gen_ai.response.model": "gpt-4o-2024"}) + + +def by_name(exporter: InMemorySpanExporter) -> dict[str, ReadableSpan]: + return {span.name: span for span in exporter.get_finished_spans()} + + +def carries_content(span: ReadableSpan) -> bool: + return any(_SECRET in str(value) for value in span.attributes.values()) + + +class TestCaptureMessageContent: + """A destination's explicit capture mode overrides the global default.""" + + @staticmethod + def _fan_out( + operator: InMemorySpanExporter, + exporters: Mapping[str, InMemorySpanExporter], + default_capture: CaptureMessageContent = CaptureMessageContent.SPAN_ONLY, + ) -> TracerProvider: + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(operator)) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda d: SimpleSpanProcessor(exporters[d.endpoint]), + default_capture=default_capture, + ) + ) + return provider + + @staticmethod + def _run( + provider: TracerProvider, destinations: tuple[OtelDestination, ...], attributes: Mapping[str, object] + ) -> None: + def run() -> None: + set_request_destinations(destinations) + model_call_tree(provider, attributes) + + in_fresh_context(run) + + @pytest.mark.parametrize( + ("setting", "content_exported"), + [ + (None, True), + ("no_content", False), + ("span_only", True), + ("event_only", False), + ("span_and_event", True), + ], + ) + def test_a_destination_setting_controls_its_copy_and_omission_uses_the_global_default( + self, setting: str | None, content_exported: bool + ) -> None: + attributes: Final = mapped(_MODEL_CALL_WITH_CONTENT) + destination: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + capture_message_content=setting, + ) + operator, tenant = InMemorySpanExporter(), InMemorySpanExporter() + + self._run(self._fan_out(operator, {destination.endpoint: tenant}), (destination,), attributes) + + exported: Final = by_name(tenant)["chat gpt-4o"] + assert carries_content(exported) is content_exported + assert carries_content(by_name(operator)["chat gpt-4o"]) + assert exported.attributes["gen_ai.request.model"] == "gpt-4o" + assert exported.attributes["gen_ai.usage.input_tokens"] == 12 + assert exported.attributes["gen_ai.usage.output_tokens"] == 8 + + def test_a_span_only_team_lifts_global_no_content_only_for_its_destination(self) -> None: + operator_exporter, team_exporter, sibling_exporter = ( + InMemorySpanExporter(), + InMemorySpanExporter(), + InMemorySpanExporter(), + ) + kind: Final = "lit8244_global_no_content_capture" + register_exporter_factory(kind, lambda _spec: operator_exporter) + config: Final = OpenTelemetryV2Config( + capture_message_content=CaptureMessageContent.NO_CONTENT, + exporters=[ExporterSpec(kind=kind)], + ) + provider: Final = build_tracer_provider(config, use_simple_processor=True, tenant_overrides=True) + team: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + capture_message_content="span_only", + ) + sibling: Final = INHERITING_DEST + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda destination: SimpleSpanProcessor( + team_exporter if destination.endpoint == team.endpoint else sibling_exporter + ), + default_capture=config.capture_message_content, + ) + ) + + self._run( + provider, + (team, sibling), + mapped(_MODEL_CALL_WITH_CONTENT), + ) + + assert carries_content(by_name(team_exporter)["chat gpt-4o"]) + assert not any(carries_content(span) for span in operator_exporter.get_finished_spans()) + assert not any(carries_content(span) for span in sibling_exporter.get_finished_spans()) + + @pytest.mark.parametrize( + ("with_content", "without_content"), + [ + (_MODEL_CALL_WITH_CONTENT, _MODEL_CALL_WITHOUT_CONTENT), + (_TOOL_CALL_WITH_CONTENT, _TOOL_CALL_WITHOUT_CONTENT), + ], + ids=["model_call", "mcp_tool_call"], + ) + def test_no_content_removes_exactly_what_global_capture_adds_in_every_mapper_vocabulary( + self, + with_content: LLMCallSpanData | MCPToolCallSpanData, + without_content: LLMCallSpanData | MCPToolCallSpanData, + ) -> None: + captured, uncaptured = mapped(with_content), mapped(without_content) + assert frozenset(captured) - frozenset(uncaptured), "the fixture must exercise captured content" + operator, tenant = InMemorySpanExporter(), InMemorySpanExporter() + + self._run(self._fan_out(operator, {NO_CONTENT_DEST.endpoint: tenant}), (NO_CONTENT_DEST,), captured) + + exported: Final = by_name(tenant)["chat gpt-4o"] + assert dict(exported.attributes) == recorded(uncaptured), "only content goes, every other attribute stays" + assert not carries_content(exported) + + def test_restricting_one_destination_leaves_the_original_span_and_the_other_destination_alone(self) -> None: + attributes: Final = mapped(_MODEL_CALL_WITH_CONTENT) + operator, restricted, inheriting = InMemorySpanExporter(), InMemorySpanExporter(), InMemorySpanExporter() + exporters: Final = {NO_CONTENT_DEST.endpoint: restricted, INHERITING_DEST.endpoint: inheriting} + + self._run(self._fan_out(operator, exporters), (NO_CONTENT_DEST, INHERITING_DEST), attributes) + + original: Final = by_name(operator) + kept: Final = by_name(inheriting) + stripped: Final = by_name(restricted) + assert dict(original["chat gpt-4o"].attributes) == recorded(attributes) + assert dict(kept["chat gpt-4o"].attributes) == recorded(attributes) + assert not carries_content(stripped["chat gpt-4o"]) + for copy in (kept, stripped): + for name, span in copy.items(): + assert span.context == original[name].context, "same trace id and span id" + assert span.parent == original[name].parent, "same place in the tree" + assert [e.name for e in copy["chat gpt-4o"].events] == [e.name for e in original["chat gpt-4o"].events] + + def test_an_omitted_setting_and_span_only_export_the_same_span(self) -> None: + attributes: Final = mapped(_MODEL_CALL_WITH_CONTENT) + span_only: Final = INHERITING_DEST.model_copy( + update={"capture_message_content": CaptureMessageContent.SPAN_ONLY} + ) + omitted_exporter, span_only_exporter = InMemorySpanExporter(), InMemorySpanExporter() + + for destination, exporter in ((INHERITING_DEST, omitted_exporter), (span_only, span_only_exporter)): + self._run( + self._fan_out(InMemorySpanExporter(), {destination.endpoint: exporter}), (destination,), attributes + ) + + assert {n: dict(s.attributes) for n, s in by_name(omitted_exporter).items()} == { + n: dict(s.attributes) for n, s in by_name(span_only_exporter).items() + } + + @pytest.mark.parametrize( + ("globally_captured", "setting", "content_exported"), + [ + (True, "no_content", False), + (True, None, True), + (False, "span_only", True), + (False, None, False), + ], + ) + def test_the_operators_exporter_on_the_same_account_honors_the_teams_setting_once( + self, + monkeypatch: pytest.MonkeyPatch, + globally_captured: bool, + setting: str | None, + content_exported: bool, + ) -> None: + """The fan-out skips a destination the operator already writes to, so the operator's + copy is the one that account receives and must not bypass the team's restriction.""" + monkeypatch.setattr(litellm, "otel_tenant_destination_mode", "additive", raising=False) + shared: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor( + _OverriddenBackendFilter( + SimpleSpanProcessor(shared), + "langfuse_otel", + "full", + TestRoutingMode.OPERATOR_SINK, + global_captures=globally_captured, + ) + ) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(shared), + operator_sinks=MappingProxyType({TestRoutingMode.OPERATOR_SINK: "full"}), + ) + ) + destination: Final = OtelDestination( + endpoint=TestRoutingMode.SAME_ACCOUNT_ENDPOINT, + headers=MappingProxyType({"Authorization": "Basic op"}), + callback_name="langfuse_otel", + capture_message_content=setting, + ) + + self._run(provider, (destination,), mapped(_MODEL_CALL_WITH_CONTENT)) + + model_calls: Final = [s for s in shared.get_finished_spans() if s.name == "chat gpt-4o"] + assert len(model_calls) == 1, "the same account received the span twice" + assert carries_content(model_calls[0]) is content_exported + + @pytest.mark.parametrize("mode", ["override", "additive"]) + def test_an_operator_collector_on_the_teams_account_does_not_bypass_no_content( + self, monkeypatch: pytest.MonkeyPatch, mode: str + ) -> None: + monkeypatch.setattr(litellm, "otel_tenant_destination_mode", mode, raising=False) + shared: Final = InMemorySpanExporter() + kind: Final = f"lit8244_collector_{mode}" + register_exporter_factory(kind, lambda _spec: shared) + config: Final = OpenTelemetryV2Config( + exporters=[ + ExporterSpec( + kind=kind, + endpoint=TestRoutingMode.OPERATOR_SINK[0], + headers="authorization=Basic op", + ) + ] + ) + provider: Final = build_tracer_provider(config, use_simple_processor=True, tenant_overrides=True) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(shared), + operator_sinks=operator_sink_scopes(config), + ) + ) + destination: Final = OtelDestination( + endpoint=TestRoutingMode.SAME_ACCOUNT_ENDPOINT, + headers=MappingProxyType({"Authorization": "Basic op"}), + callback_name="langfuse_otel", + capture_message_content="no_content", + ) + + self._run(provider, (destination,), mapped(_MODEL_CALL_WITH_CONTENT)) + + model_calls: Final = [s for s in shared.get_finished_spans() if s.name == "chat gpt-4o"] + assert model_calls, "the account must receive the model call before its content can be judged absent" + assert not any(carries_content(span) for span in model_calls) + + @pytest.mark.parametrize( + ("global_capture", "content_exported"), + [(CaptureMessageContent.NO_CONTENT, False), (CaptureMessageContent.SPAN_ONLY, True)], + ) + def test_a_routed_provider_exports_content_only_when_the_global_setting_captures( + self, global_capture: str, content_exported: bool + ) -> None: + """A sibling destination's span_only makes the logger collect content, and a request + routed to this callback's team credentials must not carry it under a global no_content.""" + exporters: dict[ExporterOwner | None, InMemorySpanExporter] = {} + kind: Final = f"lit8244_routed_{global_capture}" + register_exporter_factory(kind, lambda spec: exporters.setdefault(spec.owner, InMemorySpanExporter())) + config: Final = OpenTelemetryV2Config( + capture_message_content=global_capture, + exporters=[ + ExporterSpec( + kind=kind, + endpoint="http://op.local", + owner=ExporterOwner.LANGFUSE_OTEL, + use_simple_processor=True, + ), + ExporterSpec(kind=kind), + ], + ) + cache: Final = TenantTracerCache(config, "langfuse_otel", "litellm") + + route: Final = cache.route_for( + get_tracer(TracerProvider(), "litellm"), + {"langfuse_public_key": "pk-team", "langfuse_secret_key": "sk-team"}, + ) + with route.tracer.start_as_current_span("chat gpt-4o") as llm: + llm.set_attributes(mapped(_MODEL_CALL_WITH_CONTENT)) + cache.release(route.provider) + + assert route.detached is True + for exporter in (exporters[ExporterOwner.LANGFUSE_OTEL], exporters[None]): + copy = by_name(exporter)["chat gpt-4o"] + assert carries_content(copy) is content_exported + assert copy.attributes["gen_ai.usage.input_tokens"] == 12 + + @pytest.mark.parametrize("first", ["newrelic", "langfuse_otel"]) + def test_an_omitted_setting_follows_its_own_callbacks_mode_whatever_the_callback_order( + self, monkeypatch: pytest.MonkeyPatch, first: str + ) -> None: + monkeypatch.delenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", raising=False) + tenant: Final = InMemorySpanExporter() + monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, "otlp_http", lambda _spec: tenant) + langfuse: Final = OpenTelemetryV2Config( + capture_message_content=CaptureMessageContent.SPAN_ONLY, + exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)], + ) + newrelic: Final = OpenTelemetryV2Config( + capture_message_content=CaptureMessageContent.NO_CONTENT, + exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.NEWRELIC)], + ) + provider: Final = TracerProvider() + attach_tenant_fan_out(provider, *((newrelic, langfuse) if first == "newrelic" else (langfuse, newrelic))) + destination: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + ) + + self._run(provider, (destination,), mapped(_MODEL_CALL_WITH_CONTENT)) + provider.force_flush() + + assert carries_content(by_name(tenant)["chat gpt-4o"]) + + def test_an_omitted_setting_follows_the_capture_mode_set_in_callback_settings_otel( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", raising=False) + tenant: Final = InMemorySpanExporter() + monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, "otlp_http", lambda _spec: tenant) + operator: Final = OpenTelemetryV2Config(**{"capture_message_content": "span_only", "exporter": "in_memory"}) + provider: Final = TracerProvider() + attach_tenant_fan_out(provider, operator) + destination: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + ) + + self._run(provider, (destination,), mapped(_MODEL_CALL_WITH_CONTENT)) + provider.force_flush() + + assert carries_content(by_name(tenant)["chat gpt-4o"]), "the YAML value is the global default, not only the env" + + def test_a_published_presets_capture_mode_is_the_default_for_an_unowned_destination( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A destination for a backend the proxy does not export to itself falls back to the published logger's mode, + here New Relic with record_content on, so it keeps the content.""" + monkeypatch.delenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", raising=False) + tenant: Final = InMemorySpanExporter() + monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, "otlp_http", lambda _spec: tenant) + newrelic: Final = OpenTelemetryV2Config( + capture_message_content=CaptureMessageContent.SPAN_ONLY, + exporters=[ + ExporterSpec(kind="console"), + ExporterSpec(kind="in_memory", owner=ExporterOwner.NEWRELIC), + ], + ) + provider: Final = TracerProvider() + attach_tenant_fan_out(provider, newrelic) + destination: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + ) + + self._run(provider, (destination,), mapped(_MODEL_CALL_WITH_CONTENT)) + provider.force_flush() + + assert carries_content(by_name(tenant)["chat gpt-4o"]) + + @pytest.mark.usefixtures("allow_test_hosts") + @pytest.mark.parametrize("setting", ["no_content", "span_only", None]) + def test_the_setting_rides_the_teams_destination_and_omission_stays_omitted( + self, monkeypatch: pytest.MonkeyPatch, setting: str | None + ) -> None: + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + callback_vars: Final = { + "langfuse_public_key": "pk-team", + "langfuse_secret_key": "sk-team", + "langfuse_host": "http://team.local", + } + if setting is not None: + callback_vars["capture_message_content"] = setting + auth: Final = UserAPIKeyAuth( + team_metadata={ + "logging": [ + {"callback_name": "langfuse_otel", "callback_type": "success", "callback_vars": callback_vars}, + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {"arize_api_key": "k", "arize_space_id": "s"}, + }, + ] + } + ) + + destinations: Final = {d.callback_name: d for d in resolve_tenant_otel_destinations(auth)} + + assert destinations["langfuse_otel"].capture_message_content == setting + assert destinations["arize"].capture_message_content is None, "the setting stays on its own callback" + + @pytest.mark.usefixtures("allow_test_hosts") + @pytest.mark.parametrize("setting", ["event_only", "span_and_event"]) + def test_a_stored_event_mode_is_skipped_like_any_invalid_entry( + self, monkeypatch: pytest.MonkeyPatch, setting: str + ) -> None: + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + auth: Final = UserAPIKeyAuth( + team_metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-team", + "langfuse_secret_key": "sk-team", + "langfuse_host": "http://team.local", + "capture_message_content": setting, + }, + }, + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {"arize_api_key": "k", "arize_space_id": "s"}, + }, + ] + } + ) + + destinations: Final = [d.callback_name for d in resolve_tenant_otel_destinations(auth)] + + assert destinations == ["arize"] + + @pytest.mark.parametrize("value", ["full", "NO_CONTENT", "", "true", "event_only", "span_and_event"]) + def test_an_unsupported_value_fails_registration(self, value: str) -> None: + with pytest.raises( + ValueError, + match=r"Invalid capture_message_content .*\['no_content', 'span_only'\]", + ): + AddTeamCallback( + callback_name="langfuse_otel", + callback_type="success", + callback_vars={ + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "capture_message_content": value, + }, + ) + + @pytest.mark.parametrize("value", ["no_content", "span_only"]) + def test_a_supported_value_is_stored_as_given(self, value: str) -> None: + saved: Final = AddTeamCallback( + callback_name="langfuse_otel", + callback_type="success", + callback_vars={"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "capture_message_content": value}, + ) + + assert saved.callback_vars["capture_message_content"] == value + + def test_a_team_wide_callback_vars_map_cannot_carry_the_setting(self) -> None: + """The flattened map is shared by every callback, so a value there would apply to all of them.""" + with pytest.raises(ValueError, match="Invalid callback variable: capture_message_content"): + TeamCallbackMetadata( + success_callback=["langfuse_otel", "signoz"], callback_vars={"capture_message_content": "span_only"} + ) + + def test_flattening_the_entries_leaves_the_setting_on_its_own_entry(self) -> None: + flattened: Final = convert_key_logging_metadata_to_callback( + AddTeamCallback( + callback_name="langfuse_otel", + callback_type="success", + callback_vars={"langfuse_public_key": "pk", "capture_message_content": "span_only"}, + ), + None, + ) + + assert flattened.callback_vars == {"langfuse_public_key": "pk"} + + class TestArizeProjectRouting: """Arize rejects an export that names no project (400: set x-project-name, arize.project.name, openinference.project.name, or model_id), so an Arize diff --git a/tests/unit/integrations/otel/test_otel_v2_logger.py b/tests/unit/integrations/otel/test_otel_v2_logger.py index 3b1dd592d08..6a9a963948e 100644 --- a/tests/unit/integrations/otel/test_otel_v2_logger.py +++ b/tests/unit/integrations/otel/test_otel_v2_logger.py @@ -9,11 +9,13 @@ hooks, proxy SERVER span lifecycle (start + setters), parent-context resolution import ast import asyncio import contextlib +import contextvars import os import re from dataclasses import dataclass from datetime import datetime, timedelta, timezone from pathlib import Path +from types import MappingProxyType from typing import Final from unittest.mock import patch @@ -43,7 +45,8 @@ from litellm.integrations.otel import ( # noqa: E402 OpenTelemetryV2Config, ) from litellm.integrations.otel.logger import OpenTelemetryV2 # noqa: E402 -from litellm.integrations.otel.model.config import ExporterSpec # noqa: E402 +from litellm.integrations.otel.model.config import CaptureMessageContent, ExporterSpec # noqa: E402 +from litellm.integrations.otel.model.destination import OtelDestination # noqa: E402 from litellm.integrations.otel.model.spans import ( # noqa: E402 LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole, @@ -55,6 +58,7 @@ from litellm.integrations.otel.plumbing.context import ( # noqa: E402 reset_mcp_message_transport_span, set_mcp_message_trace_carrier, set_mcp_message_transport_span, + set_request_destinations, set_request_root_span, ) from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN # noqa: E402 @@ -617,8 +621,6 @@ def _mcp_payload(**overrides): def _logger_capturing(): - from litellm.integrations.otel.model.config import CaptureMessageContent - cfg = OpenTelemetryV2Config( exporter="in_memory", legacy_compat=False, @@ -684,6 +686,57 @@ def test_mcp_tool_call_captures_io_when_enabled(): assert "21" in span.attributes["gen_ai.tool.call.result"] +@pytest.mark.parametrize( + ("capture_mode", "captures_content"), + [ + (None, False), + (CaptureMessageContent.NO_CONTENT, False), + (CaptureMessageContent.SPAN_ONLY, True), + (CaptureMessageContent.EVENT_ONLY, False), + (CaptureMessageContent.SPAN_AND_EVENT, True), + ], +) +def test_only_a_request_destination_can_turn_on_llm_payload_content( + capture_mode: CaptureMessageContent | None, captures_content: bool +) -> None: + config: Final = OpenTelemetryV2Config( + exporter="in_memory", + capture_message_content=CaptureMessageContent.NO_CONTENT, + ) + exporter: Final = InMemorySpanExporter() + logger: Final = OpenTelemetryV2( + config=config, tracer_provider=providers.build_tracer_provider(config, exporter=exporter) + ) + payload: Final = _payload( + messages=[{"role": "user", "content": "prompt marker"}], + response={ + "id": "resp_1", + "model": "gpt-4o-2024", + "choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "answer marker"}}], + }, + ) + kwargs: Final = { + **_kwargs(payload), + "standard_callback_dynamic_params": {"capture_message_content": "span_only"}, + } + destination: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + capture_message_content=capture_mode, + ) + + def emit() -> None: + set_request_destinations((destination,)) + _emit_llm(logger, kwargs) + + contextvars.copy_context().run(emit) + + (span,) = exporter.get_finished_spans() + assert ("gen_ai.input.messages" in span.attributes) is captures_content + assert ("gen_ai.output.messages" in span.attributes) is captures_content + + def test_mcp_tool_call_failure_marks_error(): logger, exporter = _logger() payload = _mcp_payload( diff --git a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py index ea0416cecbd..318859cb3ed 100644 --- a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py @@ -1692,6 +1692,19 @@ def test_capture_message_content_normalizer_only_touches_strings(): OpenTelemetryV2Config(capture_message_content=123) +def test_unknown_capture_message_content_env_is_kept_as_a_plain_string_that_captures_nothing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "Invalid-Mode") + + config: Final = OpenTelemetryV2Config() + + assert type(config.capture_message_content) is str + assert config.capture_message_content == "invalid-mode" + assert config.capture_span_content is False + assert config.model_dump()["capture_message_content"] == "invalid-mode" + + def test_v2_flag_is_off_by_default(monkeypatch): monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) is_otel_v2_enabled.cache_clear() diff --git a/tests/unit/proxy/common_utils/test_callback_config_validation.py b/tests/unit/proxy/common_utils/test_callback_config_validation.py index d10a96e2aef..a6544e56c57 100644 --- a/tests/unit/proxy/common_utils/test_callback_config_validation.py +++ b/tests/unit/proxy/common_utils/test_callback_config_validation.py @@ -1,7 +1,11 @@ +from collections.abc import Iterator +from typing import Final + import pytest from litellm.proxy.common_utils.callback_config_validation import ( callback_config_error, + conflicting_capture_error, conflicting_span_scope_error, cross_entry_family_error, logging_metadata_config_error, @@ -259,3 +263,114 @@ def test_arize_sampling_rates_are_not_family_credentials(): stored = [{"arize_api_key": "k1", "arize_success_sampling_rate": "0.5"}] assert cross_entry_family_error({"arize_success_sampling_rate": "0.1"}, stored) is None assert cross_entry_family_error({"arize_error_sampling_rate": "0.5"}, stored) is None + + +@pytest.fixture +def otel_v2_on(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + yield + is_otel_v2_enabled.cache_clear() + + +@pytest.mark.usefixtures("otel_v2_on") +@pytest.mark.parametrize("callback_name", ["langfuse_otel", "arize", "weave_otel", "newrelic"]) +@pytest.mark.parametrize("value", ["no_content", "span_only"]) +def test_capture_message_content_is_accepted_on_every_otel_v2_destination(callback_name: str, value: str) -> None: + assert callback_config_error(callback_name, {"capture_message_content": value}) is None + + +def test_capture_message_content_is_rejected_while_otel_v2_is_off(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + try: + error: Final = callback_config_error("langfuse_otel", {"capture_message_content": "no_content"}) + finally: + is_otel_v2_enabled.cache_clear() + assert error is not None and "capture_message_content" in error and "LITELLM_OTEL_V2" in error + + +@pytest.mark.parametrize("callback_name", ["langfuse", "datadog", "otel", "arize_phoenix", None]) +def test_capture_message_content_is_rejected_where_it_would_never_take_effect(callback_name: str | None) -> None: + error: Final = callback_config_error(callback_name, {"capture_message_content": "no_content"}) + assert error is not None and "capture_message_content" in error and "langfuse_otel" in error + + +@pytest.mark.usefixtures("otel_v2_on") +@pytest.mark.parametrize("value", ["full", "event_only", "span_and_event"]) +def test_an_unsupported_capture_message_content_is_rejected_on_key_logging_metadata(value: str) -> None: + error: Final = logging_metadata_config_error( + { + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {"capture_message_content": value}, + } + ] + } + ) + assert error is not None and "Invalid capture_message_content" in error + + +@pytest.mark.usefixtures("otel_v2_on") +def test_each_entry_keeps_its_own_capture_message_content() -> None: + error: Final = logging_metadata_config_error( + { + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {"capture_message_content": "span_only"}, + }, + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {"capture_message_content": "no_content"}, + }, + ] + } + ) + assert error is None + + +@pytest.mark.usefixtures("otel_v2_on") +def test_a_failure_only_entry_rejects_capture_message_content() -> None: + error: Final = callback_config_error("langfuse_otel", {"capture_message_content": "no_content"}, "failure") + assert error is not None and "capture_message_content" in error and "success_and_failure" in error + + +@pytest.mark.parametrize( + ("stored", "rejected"), + [ + ([("langfuse_otel", {"capture_message_content": "no_content"})], True), + ([("langfuse_otel", {"capture_message_content": "span_only"})], False), + ([("langfuse_otel", {"langfuse_public_key": "pk"})], False), + ([("newrelic", {"capture_message_content": "no_content"})], False), + ], +) +def test_entries_for_one_backend_share_one_capture_message_content( + stored: list[tuple[str, dict[str, str]]], rejected: bool +) -> None: + """The backend's entries merge into one destination, so a second value would silently win or lose.""" + error: Final = conflicting_capture_error("langfuse_otel", {"capture_message_content": "span_only"}, stored) + assert (error is not None) is rejected + + +@pytest.mark.usefixtures("otel_v2_on") +def test_key_logging_entries_for_one_backend_may_not_disagree_on_capture_message_content() -> None: + def entry(callback_type: str, capture: str) -> dict[str, str | dict[str, str]]: + return { + "callback_name": "langfuse_otel", + "callback_type": callback_type, + "callback_vars": {"capture_message_content": capture}, + } + + error: Final = logging_metadata_config_error( + {"logging": [entry("success", "no_content"), entry("success_and_failure", "span_only")]} + ) + assert error is not None and "already set to 'no_content' by another langfuse_otel entry" in error diff --git a/tests/unit/proxy/management_endpoints/test_team_callback_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_callback_endpoints.py index f576bac4742..ad5b6903ece 100644 --- a/tests/unit/proxy/management_endpoints/test_team_callback_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_callback_endpoints.py @@ -7,6 +7,7 @@ redacted audit rows for callback mutations. """ import json +from collections.abc import Iterator from typing import Final from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -1712,3 +1713,62 @@ def test_add_team_callback_accepts_arize_sampling_rate_vars(): ) assert data.callback_vars["arize_success_sampling_rate"] == "0.5" assert data.callback_vars["arize_error_sampling_rate"] == "1.0" + + +@pytest.mark.asyncio +async def test_add_team_callbacks_rejects_capture_message_content_on_a_non_otel_v2_callback(patched_prisma): + data: Final = AddTeamCallback( + callback_name="langfuse", + callback_type="success", + callback_vars={ + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "capture_message_content": "no_content", + }, + ) + with pytest.raises(HTTPException) as exc: + await add_team_callbacks( + data=data, + http_request=Mock(spec=Request), + team_id="team-victim", + user_api_key_dict=_admin_auth(), + ) + assert exc.value.status_code == 400 + assert "capture_message_content" in str(exc.value.detail) + patched_prisma.db.litellm_teamtable.update.assert_not_called() + + +@pytest.fixture +def otel_v2_on(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + yield + is_otel_v2_enabled.cache_clear() + + +@pytest.mark.usefixtures("otel_v2_on") +@pytest.mark.asyncio +@pytest.mark.parametrize("capture", ["no_content", "span_only"]) +async def test_add_team_callbacks_stores_capture_message_content_in_the_existing_callback_metadata( + patched_prisma, capture: str +) -> None: + data: Final = AddTeamCallback( + callback_name="langfuse_otel", + callback_type="success", + callback_vars={ + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "capture_message_content": capture, + }, + ) + await add_team_callbacks( + data=data, + http_request=Mock(spec=Request), + team_id="team-victim", + user_api_key_dict=_admin_auth(), + ) + patched_prisma.db.litellm_teamtable.update.assert_awaited_once() + stored: Final = json.loads(patched_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"]) + assert [entry["callback_vars"].get("capture_message_content") for entry in stored["logging"]] == [capture] diff --git a/tests/unit/types/test_utils.py b/tests/unit/types/test_utils.py new file mode 100644 index 00000000000..09063d894e7 --- /dev/null +++ b/tests/unit/types/test_utils.py @@ -0,0 +1,40 @@ +import pytest + +from litellm.types.utils import CAPTURE_MESSAGE_CONTENT_VALUES, CaptureMessageContent, captures_span_content + + +@pytest.mark.parametrize( + "mode", + [ + CaptureMessageContent.NO_CONTENT, + CaptureMessageContent.SPAN_ONLY, + CaptureMessageContent.EVENT_ONLY, + CaptureMessageContent.SPAN_AND_EVENT, + ], +) +def test_capture_message_content_members_are_plain_strings(mode: str) -> None: + assert type(mode) is str + assert (str(mode), f"{mode}", repr(mode)) == (mode, mode, repr(str(mode))) + + +def test_a_destination_may_only_set_the_span_capture_modes() -> None: + assert CAPTURE_MESSAGE_CONTENT_VALUES == {"no_content", "span_only"} + + +def test_capture_message_content_constructor_accepts_any_string() -> None: + assert CaptureMessageContent("custom") == "custom" + + +@pytest.mark.parametrize( + ("mode", "captures"), + [ + ("no_content", False), + ("span_only", True), + ("event_only", False), + ("span_and_event", True), + ("bogus", False), + (None, False), + ], +) +def test_captures_span_content(mode: str | None, captures: bool) -> None: + assert captures_span_content(mode) is captures diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 731a92d8b98..47df4593a65 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -17246,6 +17246,9 @@ export interface paths { * - langfuse_host: The host for the Langfuse callback * - langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT) * - langfuse_span_scope: For langfuse_otel, "full" (default) sends the whole request trace, "llm_only" sends only the model-call spans + * - capture_message_content: For OTel v2 callbacks, "no_content" or "span_only". + * An explicit value overrides the global setting for this destination; omitted follows the proxy's own callback for that backend, else the global setting. + * "span_only" puts prompt and response content on the destination's spans, "no_content" leaves it out * - gcs_bucket_name: The name of the GCS bucket * - gcs_path_service_account: The path to the GCS service account * - langsmith_api_key: The API key for the Langsmith callback From c9d706adb5d387807d9d03fd9d90c49ef8a66847 Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 13:58:07 -0700 Subject: [PATCH 13/40] refactor(types): move key request models into litellm.types.proxy.management_endpoints.key_requests (#45810) * refactor(types): move LitellmUserRoles and KeyManagementRoutes into litellm.types.proxy.auth.user_roles Part of LIT-9423 * refactor(types): annotate GenerateRequestBase containers and check_max_budget Part of LIT-9423 * refactor(types): move GenerateRequestBase and LiteLLM_ObjectPermissionBase into litellm.types.proxy.management_endpoints.request_base Part of LIT-9423 * refactor(types): annotate key request containers and drop_blank_team_id Part of LIT-9423 * refactor(types): move key request models into litellm.types.proxy.management_endpoints.key_requests Part of LIT-9423 * fix(types): keep MappingProxyType exported from litellm.proxy._types for star importers Part of LIT-9423 * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main --- litellm/proxy/_types.py | 144 +++--------------- .../key_management_endpoints.py | 8 +- .../management_endpoints/key_requests.py | 138 +++++++++++++++++ 3 files changed, 161 insertions(+), 129 deletions(-) create mode 100644 litellm/types/proxy/management_endpoints/key_requests.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 39892bcc29c..4cbf4f15d6d 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3,7 +3,7 @@ import json import os from collections.abc import Callable, Mapping, Sequence from datetime import datetime -from types import MappingProxyType +from types import MappingProxyType as MappingProxyType # noqa: PLC0414 # public re-export from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NamedTuple, TypeAlias import httpx @@ -1165,6 +1165,18 @@ from litellm.models.team import BudgetLimitEntry as BudgetLimitEntry # noqa: E4 from litellm.types.object_permission import ( # noqa: E402 ObjectPermissionDict as ObjectPermissionDict, ) +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + AllowedVectorStoreIndexItem as AllowedVectorStoreIndexItem, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + GenerateKeyRequest as GenerateKeyRequest, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + KeyRequestBase as KeyRequestBase, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + LiteLLMKeyType as LiteLLMKeyType, # noqa: PLC0414 # public re-export +) from litellm.types.proxy.management_endpoints.request_base import ( # noqa: E402 # re-export after the definitions above GenerateRequestBase as GenerateRequestBase, # noqa: PLC0414 # public re-export ) @@ -1173,71 +1185,6 @@ from litellm.types.proxy.management_endpoints.request_base import ( # noqa: E40 ) -class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase): - index_name: str - index_permissions: list[Literal["read", "write"]] - - -class KeyRequestBase(GenerateRequestBase): - key: str | None = None - tpd_limit: int | None = None - default_estimated_output_tokens: PositiveInt | None = None - default_estimated_output_tokens_per_model: Mapping[str, PositiveInt] | None = None - budget_id: str | None = None - end_user_budget_id: str | None = None - tags: list[str] | None = None - disable_global_guardrails: bool | None = None - enable_prompt_caching: bool | None = None - throttle_on_budget_exceeded: bool | None = None - enforced_params: list[str] | None = None - allowed_routes: list[Any] | None = [] - allowed_passthrough_routes: list[object] | None = None - denied_passthrough_routes: list[str] | None = None - allowed_vector_store_indexes: list[AllowedVectorStoreIndexItem] | None = None - rpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = ( - None # raise an error if 'guaranteed_throughput' is set and we're overallocating rpm - ) - tpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = ( - None # raise an error if 'guaranteed_throughput' is set and we're overallocating tpm - ) - router_settings: UpdateRouterConfig | None = None - access_group_ids: list[str] | None = None - - -class LiteLLMKeyType(str, enum.Enum): - """ - Enum for key types that determine what routes a key can access - """ - - LLM_API = "llm_api" # Can call LLM API routes (chat/completions, embeddings, etc.) - MANAGEMENT = "management" # Can call management routes (user/team/key management) - READ_ONLY = "read_only" # Can only call info/read routes - DEFAULT = "default" # Uses default allowed routes - - -class GenerateKeyRequest(KeyRequestBase): - soft_budget: float | None = None - send_invite_email: bool | None = None - key_type: LiteLLMKeyType | None = Field( - default=LiteLLMKeyType.DEFAULT, - description="Type of key that determines default allowed routes.", - ) - auto_rotate: bool | None = Field(default=False, description="Whether this key should be automatically rotated") - rotation_interval: str | None = Field( - default=None, - description="How often to rotate this key (e.g., '30d', '90d'). Required if auto_rotate=True", - ) - organization_id: str | None = None - project_id: str | None = None - - @field_validator("team_id", "organization_id", "project_id", mode="before") - @classmethod - def treat_cleared_id_as_unset(cls, v: object) -> object: - if v == "": - return None - return v - - class GenerateKeyResponse(KeyRequestBase): key: str key_name: str | None = None @@ -1280,65 +1227,12 @@ class GenerateKeyResponse(KeyRequestBase): return values -def _is_mapping(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: raw before-validator payload - return isinstance(value, Mapping) - - -class UpdateKeyRequest(KeyRequestBase): - # Note: the defaults of all Params here MUST BE NONE - # else they will get overwritten - duration: str | None = None - spend: float | None = None - soft_budget: float | None = None - metadata: dict[Any, Any] | None = None - temp_budget_increase: float | None = None - temp_budget_expiry: datetime | None = None - auto_rotate: bool | None = None - rotation_interval: str | None = None - organization_id: str | None = None - - project_id: str | None = Field( - default=None, - description="Omit to retain the project, or send null to detach. Assigning a different project is not supported.", - ) - - @model_validator(mode="before") - @classmethod - def drop_blank_team_id(cls, values: object) -> object: - if _is_mapping(values) and values.get("team_id") == "": - return MappingProxyType({k: v for k, v in values.items() if k != "team_id"}) - return values - - @field_validator("organization_id", mode="before") - @classmethod - def treat_cleared_organization_id_as_unset(cls, v: object) -> object: - if v == "": - return None - return v - - @model_validator(mode="after") - def validate_temp_budget(self) -> "UpdateKeyRequest": - if self.temp_budget_increase is not None or self.temp_budget_expiry is not None: - if self.temp_budget_increase is None or self.temp_budget_expiry is None: - raise ValueError("temp_budget_increase and temp_budget_expiry must be set together") - return self - - @model_validator(mode="after") - def validate_key_identifier(self) -> "UpdateKeyRequest": - if self.key is None and self.key_alias is None: - raise ValueError("either key or key_alias must be provided") - return self - - -class RegenerateKeyRequest(GenerateKeyRequest): - # This needs to be different from UpdateKeyRequest, because "key" is optional for this - key: str | None = None - new_key: str | None = None - duration: str | None = None - spend: float | None = None - metadata: dict[Any, Any] | None = None - new_master_key: str | None = None - grace_period: str | None = None # Duration to keep old key valid (e.g. "24h", "2d"); None = immediate revoke +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + RegenerateKeyRequest as RegenerateKeyRequest, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + UpdateKeyRequest as UpdateKeyRequest, # noqa: PLC0414 # public re-export +) class ResetSpendRequest(LiteLLMPydanticObjectBase): diff --git a/litellm/types/proxy/management_endpoints/key_management_endpoints.py b/litellm/types/proxy/management_endpoints/key_management_endpoints.py index 05b63100ea4..f4348391710 100644 --- a/litellm/types/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/key_management_endpoints.py @@ -5,14 +5,14 @@ from pydantic import ConfigDict, model_validator from typing_extensions import ReadOnly, TypedDict from litellm.models.verification_token import LiteLLM_VerificationToken -from litellm.proxy._types import ( +from litellm.types.llms.base import LiteLLMBaseModel, LiteLLMPydanticObjectBase +from litellm.types.proxy.management_endpoints.internal_user_endpoints import InsensitiveContains +from litellm.types.proxy.management_endpoints.key_requests import ( GenerateKeyRequest, - LiteLLM_ObjectPermissionBase, RegenerateKeyRequest, UpdateKeyRequest, ) -from litellm.types.llms.base import LiteLLMBaseModel, LiteLLMPydanticObjectBase -from litellm.types.proxy.management_endpoints.internal_user_endpoints import InsensitiveContains +from litellm.types.proxy.management_endpoints.request_base import LiteLLM_ObjectPermissionBase class KeyTokenWhere(TypedDict): diff --git a/litellm/types/proxy/management_endpoints/key_requests.py b/litellm/types/proxy/management_endpoints/key_requests.py new file mode 100644 index 00000000000..baa02b68d98 --- /dev/null +++ b/litellm/types/proxy/management_endpoints/key_requests.py @@ -0,0 +1,138 @@ +import enum +from collections.abc import Mapping +from datetime import datetime +from types import MappingProxyType +from typing import Any, Literal # noqa: TID251 # allowed_routes and metadata stay Any for existing callers + +from pydantic import Field, PositiveInt, field_validator, model_validator +from typing_extensions import TypeIs # noqa: TID251 # narrows the raw payload a before-validator receives + +from litellm.types.proxy.management_endpoints.request_base import GenerateRequestBase +from litellm.types.router import UpdateRouterConfig +from litellm.types.utils import LiteLLMPydanticObjectBase + + +class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase): + index_name: str + index_permissions: list[Literal["read", "write"]] + + +class KeyRequestBase(GenerateRequestBase): + key: str | None = None + tpd_limit: int | None = None + default_estimated_output_tokens: PositiveInt | None = None + default_estimated_output_tokens_per_model: Mapping[str, PositiveInt] | None = None + budget_id: str | None = None + end_user_budget_id: str | None = None + tags: list[str] | None = None + disable_global_guardrails: bool | None = None + enable_prompt_caching: bool | None = None + throttle_on_budget_exceeded: bool | None = None + enforced_params: list[str] | None = None + allowed_routes: list[Any] | None = [] + allowed_passthrough_routes: list[object] | None = None + denied_passthrough_routes: list[str] | None = None + allowed_vector_store_indexes: list[AllowedVectorStoreIndexItem] | None = None + rpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = ( + None # raise an error if 'guaranteed_throughput' is set and we're overallocating rpm + ) + tpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = ( + None # raise an error if 'guaranteed_throughput' is set and we're overallocating tpm + ) + router_settings: UpdateRouterConfig | None = None + access_group_ids: list[str] | None = None + + +class LiteLLMKeyType(str, enum.Enum): + """ + Enum for key types that determine what routes a key can access + """ + + LLM_API = "llm_api" # Can call LLM API routes (chat/completions, embeddings, etc.) + MANAGEMENT = "management" # Can call management routes (user/team/key management) + READ_ONLY = "read_only" # Can only call info/read routes + DEFAULT = "default" # Uses default allowed routes + + +class GenerateKeyRequest(KeyRequestBase): + soft_budget: float | None = None + send_invite_email: bool | None = None + key_type: LiteLLMKeyType | None = Field( + default=LiteLLMKeyType.DEFAULT, + description="Type of key that determines default allowed routes.", + ) + auto_rotate: bool | None = Field(default=False, description="Whether this key should be automatically rotated") + rotation_interval: str | None = Field( + default=None, + description="How often to rotate this key (e.g., '30d', '90d'). Required if auto_rotate=True", + ) + organization_id: str | None = None + project_id: str | None = None + + @field_validator("team_id", "organization_id", "project_id", mode="before") + @classmethod + def treat_cleared_id_as_unset(cls, v: object) -> object: + if v == "": + return None + return v + + +def _is_mapping(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: raw before-validator payload + return isinstance(value, Mapping) + + +class UpdateKeyRequest(KeyRequestBase): + # Note: the defaults of all Params here MUST BE NONE + # else they will get overwritten + duration: str | None = None + spend: float | None = None + soft_budget: float | None = None + metadata: dict[Any, Any] | None = None + temp_budget_increase: float | None = None + temp_budget_expiry: datetime | None = None + auto_rotate: bool | None = None + rotation_interval: str | None = None + organization_id: str | None = None + + project_id: str | None = Field( + default=None, + description="Omit to retain the project, or send null to detach. Assigning a different project is not supported.", + ) + + @model_validator(mode="before") + @classmethod + def drop_blank_team_id(cls, values: object) -> object: + if _is_mapping(values) and values.get("team_id") == "": + return MappingProxyType({k: v for k, v in values.items() if k != "team_id"}) + return values + + @field_validator("organization_id", mode="before") + @classmethod + def treat_cleared_organization_id_as_unset(cls, v: object) -> object: + if v == "": + return None + return v + + @model_validator(mode="after") + def validate_temp_budget(self) -> "UpdateKeyRequest": + if self.temp_budget_increase is not None or self.temp_budget_expiry is not None: + if self.temp_budget_increase is None or self.temp_budget_expiry is None: + raise ValueError("temp_budget_increase and temp_budget_expiry must be set together") + return self + + @model_validator(mode="after") + def validate_key_identifier(self) -> "UpdateKeyRequest": + if self.key is None and self.key_alias is None: + raise ValueError("either key or key_alias must be provided") + return self + + +class RegenerateKeyRequest(GenerateKeyRequest): + # This needs to be different from UpdateKeyRequest, because "key" is optional for this + key: str | None = None + new_key: str | None = None + duration: str | None = None + spend: float | None = None + metadata: dict[Any, Any] | None = None + new_master_key: str | None = None + grace_period: str | None = None # Duration to keep old key valid (e.g. "24h", "2d"); None = immediate revoke From 513e7a89ac17e5301ceb57fa30e078454cacee7f Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 13:58:11 -0700 Subject: [PATCH 14/40] refactor(types): move user request models into litellm.types.proxy.management_endpoints.user_requests (#45813) * refactor(types): move LitellmUserRoles and KeyManagementRoutes into litellm.types.proxy.auth.user_roles Part of LIT-9423 * refactor(types): annotate GenerateRequestBase containers and check_max_budget Part of LIT-9423 * refactor(types): move GenerateRequestBase and LiteLLM_ObjectPermissionBase into litellm.types.proxy.management_endpoints.request_base Part of LIT-9423 * refactor(types): annotate user request metadata and check_user_info validators Part of LIT-9423 * refactor(types): move user request models into litellm.types.proxy.management_endpoints.user_requests Part of LIT-9423 * fix(types): restore main's _types.py content alongside the request_base move Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * style(types): merge request_base re-exports into one import block Part of LIT-9423 --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 84 ++++--------------- .../internal_user_endpoints.py | 6 +- .../management_endpoints/user_requests.py | 82 ++++++++++++++++++ 3 files changed, 100 insertions(+), 72 deletions(-) create mode 100644 litellm/types/proxy/management_endpoints/user_requests.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 4cbf4f15d6d..f43ce4dc211 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1684,41 +1684,12 @@ class ListSkillsRequest(LiteLLMPydanticObjectBase): offset: int | None = 0 -class NewUserRequestTeam(LiteLLMPydanticObjectBase): - team_id: str - max_budget_in_team: float | None = None - user_role: Literal["user", "admin"] = "user" - - -class NewUserRequest(GenerateRequestBase): - max_budget: float | None = None - user_email: str | None = None - user_alias: str | None = None - user_role: ( - Literal[ - LitellmUserRoles.PROXY_ADMIN, - LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - LitellmUserRoles.INTERNAL_USER, - LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, - ] - | None - ) = None - teams: list[str] | list[NewUserRequestTeam] | None = None - auto_create_key: bool = True # flag used for returning a key as part of the /user/new response - send_invite_email: bool | None = None - sso_user_id: str | None = None - organizations: list[str] | None = None - password: str | None = None - - @field_validator("password") - @classmethod - def password_not_supported(cls, value: str | None) -> str | None: - if value is not None: - raise ValueError( - "password cannot be set via /user/new. Users set their own password through an " - "invitation link (POST /invitation/new)." - ) - return value +from litellm.types.proxy.management_endpoints.user_requests import ( # noqa: E402 # re-export after the definitions above + NewUserRequest as NewUserRequest, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.user_requests import ( # noqa: E402 # re-export after the definitions above + NewUserRequestTeam as NewUserRequestTeam, # noqa: PLC0414 # public re-export +) class NewUserResponse(GenerateKeyResponse): @@ -1740,36 +1711,12 @@ class NewUserResponse(GenerateKeyResponse): updated_at: datetime | None = None -class UpdateUserRequestNoUserIDorEmail(GenerateRequestBase): # shared with BulkUpdateUserRequest - # repr=False keeps the plaintext out of management-endpoint alerts, which str() the request model - password: str | None = Field(default=None, repr=False) - spend: float | None = None - metadata: dict[object, object] | None = None - user_alias: str | None = None - user_role: ( - Literal[ - LitellmUserRoles.PROXY_ADMIN, - LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - LitellmUserRoles.INTERNAL_USER, - LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, - ] - | None - ) = None - max_budget: float | None = None - - -class UpdateUserRequest(UpdateUserRequestNoUserIDorEmail): - # Note: the defaults of all Params here MUST BE NONE - # else they will get overwritten - user_id: str | None = None - user_email: str | None = None - - @model_validator(mode="before") - @classmethod - def check_user_info(cls, values: Mapping[str, object]) -> Mapping[str, object]: - if values.get("user_id") is None and values.get("user_email") is None: - raise ValueError("Either user id or user email must be provided") - return values +from litellm.types.proxy.management_endpoints.user_requests import ( # noqa: E402 # re-export after the definitions above + UpdateUserRequest as UpdateUserRequest, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.user_requests import ( # noqa: E402 # re-export after the definitions above + UpdateUserRequestNoUserIDorEmail as UpdateUserRequestNoUserIDorEmail, # noqa: PLC0414 # public re-export +) class ChangePasswordRequest(LiteLLMPydanticObjectBase): @@ -3632,10 +3579,6 @@ class LiteLLM_UserTableFiltered(LiteLLMBaseModel): # done to avoid exposing sen user_email: str | None = None -class LiteLLM_UserTableWithKeyCount(LiteLLM_UserTable): - key_count: int = 0 - - from litellm.models.access_group import ( # noqa: E402 LiteLLM_AccessGroupTable as LiteLLM_AccessGroupTable, ) @@ -3649,6 +3592,9 @@ from litellm.models.spend_logs import ( # noqa: E402 LiteLLM_SpendLogs as LiteLLM_SpendLogs, ) from litellm.models.tag import LiteLLM_TagTable as LiteLLM_TagTable # noqa: E402 +from litellm.types.proxy.management_endpoints.user_requests import ( # noqa: E402 # re-export after the definitions above + LiteLLM_UserTableWithKeyCount as LiteLLM_UserTableWithKeyCount, # noqa: PLC0414 # public re-export +) AUDIT_ACTIONS = Literal["created", "updated", "deleted", "blocked", "unblocked", "rotated", "kill_switch_fired"] diff --git a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py index 1fd3e7dca53..febd4485d10 100644 --- a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py @@ -4,14 +4,14 @@ from typing import Any, Final, Literal from pydantic import ConfigDict, Field, field_validator from typing_extensions import ReadOnly, TypedDict -from litellm.proxy._types import ( +from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.proxy.management_endpoints.management_v1 import ResourceResponse +from litellm.types.proxy.management_endpoints.user_requests import ( LiteLLM_UserTableWithKeyCount, NewUserRequest, UpdateUserRequest, UpdateUserRequestNoUserIDorEmail, ) -from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.proxy.management_endpoints.management_v1 import ResourceResponse MAX_BULK_DELETE_USERS: Final = 500 diff --git a/litellm/types/proxy/management_endpoints/user_requests.py b/litellm/types/proxy/management_endpoints/user_requests.py new file mode 100644 index 00000000000..49dda5995bc --- /dev/null +++ b/litellm/types/proxy/management_endpoints/user_requests.py @@ -0,0 +1,82 @@ +from collections.abc import Mapping +from typing import Literal + +from pydantic import Field, field_validator, model_validator + +from litellm.models.user import LiteLLM_UserTable +from litellm.types.proxy.auth.user_roles import LitellmUserRoles +from litellm.types.proxy.management_endpoints.request_base import GenerateRequestBase +from litellm.types.utils import LiteLLMPydanticObjectBase + + +class NewUserRequestTeam(LiteLLMPydanticObjectBase): + team_id: str + max_budget_in_team: float | None = None + user_role: Literal["user", "admin"] = "user" + + +class NewUserRequest(GenerateRequestBase): + max_budget: float | None = None + user_email: str | None = None + user_alias: str | None = None + user_role: ( + Literal[ + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ] + | None + ) = None + teams: list[str] | list[NewUserRequestTeam] | None = None + auto_create_key: bool = True # flag used for returning a key as part of the /user/new response + send_invite_email: bool | None = None + sso_user_id: str | None = None + organizations: list[str] | None = None + password: str | None = None + + @field_validator("password") + @classmethod + def password_not_supported(cls, value: str | None) -> str | None: + if value is not None: + raise ValueError( + "password cannot be set via /user/new. Users set their own password through an " + "invitation link (POST /invitation/new)." + ) + return value + + +class UpdateUserRequestNoUserIDorEmail(GenerateRequestBase): # shared with BulkUpdateUserRequest + # repr=False keeps the plaintext out of management-endpoint alerts, which str() the request model + password: str | None = Field(default=None, repr=False) + spend: float | None = None + metadata: dict[object, object] | None = None + user_alias: str | None = None + user_role: ( + Literal[ + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ] + | None + ) = None + max_budget: float | None = None + + +class UpdateUserRequest(UpdateUserRequestNoUserIDorEmail): + # Note: the defaults of all Params here MUST BE NONE + # else they will get overwritten + user_id: str | None = None + user_email: str | None = None + + @model_validator(mode="before") + @classmethod + def check_user_info(cls, values: Mapping[str, object]) -> Mapping[str, object]: + if values.get("user_id") is None and values.get("user_email") is None: + raise ValueError("Either user id or user email must be provided") + return values + + +class LiteLLM_UserTableWithKeyCount(LiteLLM_UserTable): + key_count: int = 0 From 2011004d0adf28c45d0bc2194b3cdba67f666327 Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 13:58:46 -0700 Subject: [PATCH 15/40] refactor(types): move MemberDeleteRequest into litellm.types.proxy.management_endpoints.team_endpoints (#45814) * refactor(types): move LitellmUserRoles and KeyManagementRoutes into litellm.types.proxy.auth.user_roles Part of LIT-9423 * refactor(types): annotate GenerateRequestBase containers and check_max_budget Part of LIT-9423 * refactor(types): move GenerateRequestBase and LiteLLM_ObjectPermissionBase into litellm.types.proxy.management_endpoints.request_base Part of LIT-9423 * refactor(types): annotate user request metadata and check_user_info validators Part of LIT-9423 * refactor(types): move user request models into litellm.types.proxy.management_endpoints.user_requests Part of LIT-9423 * refactor(types): move MemberDeleteRequest into litellm.types.proxy.management_endpoints.team_endpoints Part of LIT-9423 * fix(types): restore main's _types.py content alongside the request_base move Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * style(types): merge request_base re-exports into one import block Part of LIT-9423 --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 13 +++---------- .../management_endpoints/team_endpoints.py | 19 +++++++++++++++---- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f43ce4dc211..4abb93454c7 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4370,16 +4370,9 @@ class OrganizationAddMemberResponse(LiteLLMPydanticObjectBase): updated_organization_memberships: list[LiteLLM_OrganizationMembershipTable] -class MemberDeleteRequest(LiteLLMPydanticObjectBase): - user_id: str | None = None - user_email: str | None = None - - @model_validator(mode="before") - @classmethod - def check_user_info(cls, values: Mapping[str, object]) -> Mapping[str, object]: - if values.get("user_id") is None and values.get("user_email") is None: - raise ValueError("Either user id or user email must be provided") - return values +from litellm.types.proxy.management_endpoints.team_endpoints import ( # noqa: E402 # re-export after the definitions above + MemberDeleteRequest as MemberDeleteRequest, # noqa: PLC0414 # public re-export +) class MemberUpdateResponse(LiteLLMPydanticObjectBase): diff --git a/litellm/types/proxy/management_endpoints/team_endpoints.py b/litellm/types/proxy/management_endpoints/team_endpoints.py index 796bcae37bc..065205dcad8 100644 --- a/litellm/types/proxy/management_endpoints/team_endpoints.py +++ b/litellm/types/proxy/management_endpoints/team_endpoints.py @@ -1,16 +1,15 @@ +from collections.abc import Mapping from typing import Any, Final, Literal from pydantic import ConfigDict, Field, field_validator, model_validator from litellm.models.team import LiteLLM_DeletedTeamTable, LiteLLM_TeamTable, Member from litellm.models.team_membership import LiteLLM_TeamMembership -from litellm.proxy._types import ( - KeyManagementRoutes, - MemberDeleteRequest, -) from litellm.proxy.common_utils.timezone_utils import budget_duration_error from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.proxy.auth.user_roles import KeyManagementRoutes from litellm.types.proxy.management_endpoints.management_v1 import ResourceResponse +from litellm.types.utils import LiteLLMPydanticObjectBase TeamIdSearchMatch = Literal["exact", "prefix"] @@ -19,6 +18,18 @@ MAX_BULK_TEAM_MEMBER_DELETES: Final = 500 MAX_BULK_TEAM_MEMBER_BUDGET_UPDATES: Final = 500 +class MemberDeleteRequest(LiteLLMPydanticObjectBase): + user_id: str | None = None + user_email: str | None = None + + @model_validator(mode="before") + @classmethod + def check_user_info(cls, values: Mapping[str, object]) -> Mapping[str, object]: + if values.get("user_id") is None and values.get("user_email") is None: + raise ValueError("Either user id or user email must be provided") + return values + + class GetTeamMemberPermissionsRequest(LiteLLMBaseModel): """Request to get the team member permissions for a team""" From 38bdda9ca7c2f7b31d8d133a3d09f4c8a8d694b3 Mon Sep 17 00:00:00 2001 From: Soham Chatterjee <96soham96@gmail.com> Date: Sat, 10 Oct 2026 23:01:17 +0200 Subject: [PATCH 16/40] feat(scaledown): add native text provider support (#44168) * Add ScaleDown model pricing Registers the five ScaleDown models in the pricing map. ScaleDown bills on input tokens only, so output_cost_per_token is zero for every entry. The classify and decisions models bill at $0.04 per million input tokens, and extract, summarize, and compress bill at $0.05 per million. Co-Authored-By: Claude Opus 5 * Price all ScaleDown models at $0.04 per million and drop unverified context limits Every ScaleDown model now bills $0.04 per million input tokens. The 32000-token max_input_tokens/max_output_tokens values were not confirmed against the service, so they are removed rather than registered. Co-Authored-By: Claude Sonnet 5.5 * Price all ScaleDown models at $0.05 per million input tokens This is the standard rate in ScaleDown's billing and matches the usage dashboard. The Decisions usage.cost field is not used for pricing. Co-Authored-By: Claude Sonnet 5.5 * Add regression tests for the ScaleDown pricing entries Co-Authored-By: Claude Sonnet 5.5 * Cite the source of the ScaleDown rate in the pricing tests and type them Co-Authored-By: Claude Sonnet 5.5 * Add ScaleDown as a chat provider ScaleDown serves five models over two wire protocols. extract, summarize, and compress speak OpenAI chat on /v1/chat/completions, while classify and decisions implement the Jev Decisions API on /v1/scaledown, taking a state object plus a questions map and returning one typed answer per question. The provider authenticates with x-api-key rather than a bearer token, and the decisions models need a non-chat request body, so it routes through the HTTP handler instead of the OpenAI passthrough. Malformed decisions requests are rejected locally with a message naming the problem, so callers do not have to decode an opaque 422 from upstream. Decisions bills one model call per question, so the cost upstream reports is passed through rather than re-estimated from token counts. Co-Authored-By: Claude Opus 5 * Route ScaleDown domain models to the native API and stop trusting usage.cost The /v1/chat/completions and /v1/models routes return 403 on api.scaledown.xyz, so extract, summarize and compress now call the native /extract, /summarization/abstractive and /compress/raw/ endpoints. The default api_base is the bare host; a trailing /v1 is stripped and re-added only for decisions. The Decisions usage.cost field was about 600x the dashboard amount, so it is no longer passed through; cost is computed from input tokens at the registered rate. Native responses are returned unchanged as JSON, including the _value/_span_anchor wrappers on nested extraction. They carry no output token count, so completion_tokens is reported as 0 and documented as unmeasured. Co-Authored-By: Claude Sonnet 5.5 * Move ScaleDown translation into llms/ and address review findings - Dispatch moves to llms/scaledown/chat/handler.py; main.py keeps a thin call and get_llm_provider_logic.py no longer has a ScaleDown branch, since the config resolves key and base URL itself. - questions passed via extra_body are merged and validated after the shared merge (sign_request), so the documented extra_body path works. - extra_body can no longer change the upstream model, and decisions state may carry a document but not text, so guardrails always see the text. - A custom api_base requires its own api_key; the environment key is only sent to the default host or SCALEDOWN_API_BASE. - Image content parts become the decisions document; max_completion_tokens maps to max_tokens; stream is served as one chunk for every model. Co-Authored-By: Claude Sonnet 5.5 * Refuse prompt-bearing extra_body keys and empty-key fallback to the env credential extra_body is merged after guardrails inspect the messages, so each model now accepts only option keys there (extract threshold/top_n, compress compression_rate, decisions questions/state); text, instructions, prompt, context and model are rejected. An empty api_key no longer lets the SCALEDOWN_API_KEY from the environment go to a custom api_base. Co-Authored-By: Claude Sonnet 5.5 * Read compress token usage from the real response shape /compress/raw/ nests the per-prompt counts under results and reports the total as a top-level input_tokens; the provider read a top-level original_prompt_tokens that is not there, so compress reported zero tokens and zero cost. Co-Authored-By: Claude Sonnet 5.5 * Accept the router's max_retries and word the extra_body error clearly The proxy adds max_retries to every call, which map_openai_params rejected, so every proxied ScaleDown request failed with a 400. It is now accepted and not forwarded. Co-Authored-By: Claude Sonnet 5.5 * Return extracted fields from extract, follow schema refs, and meet the type-discipline gate - extract content is now the extracted fields (span anchors dropped, wrappers unwrapped), so it validates against the caller's response_format; the raw payload stays on _hidden_params['scaledown_response']. - Local $ref/$defs in the response_format schema are resolved. - The adapter is rewritten with Final locals, read-only annotations, one raising helper and a typed handler; check_type_discipline reports no violations. Co-Authored-By: Claude Sonnet 5.5 * Fix stream_options, schema-guided extract cleaning, document input and multi-image handling - stream_options is accepted and not sent upstream, so include_usage streams work. - Extract output is cleaned against the requested entities, so a field named _value or *_span_anchor is no longer dropped. - Extract and summarize send an image or PDF as the native document fields. - More than one image is rejected instead of silently using the first. - The cost test derives its rate from the cost map. Co-Authored-By: Claude Sonnet 5.5 * Cut recursive schema refs at the cycle and cap the expanded entity count A small response_format schema with a self-referencing $def expanded to 4^16 leaves before any request was sent. Refs that point back into themselves are now cut off at that point, and the total expanded entities are capped at 1000 so repeated acyclic refs are rejected with a clear error. Co-Authored-By: Claude Sonnet 5.5 * Bound ref chains and schema nesting so long schemas fail cleanly Resolving a chain of thousands of distinct $refs recursed until RecursionError. Ref resolution is now an iterative walk capped at 32 hops, and nested properties are capped at 32 levels; both raise a clear 400 instead. Co-Authored-By: Claude Sonnet 5.5 * fix(scaledown): support native classify and preserve extraction results * test(scaledown): include provider regressions in CI shard * fix(scaledown): pass registry typing and bounded recursion checks * docs(scaledown): capture provider setup before and after * fix(scaledown): preserve nullable schemas and reject malformed requests * fix(scaledown): isolate provider dispatch from existing providers * fix(scaledown): preserve the shared dispatch return contract * fix(scaledown): preserve failure logging and reject lossy schemas * fix(scaledown): keep timeout typing annotation on its expression * fix(scaledown): resolve escaped extraction schema references * fix(scaledown): annotate the bounded reference walk * chore(scaledown): remove unused loop suppression --------- Co-authored-by: Claude Opus 5 Co-authored-by: moe-berri --- .github/assets/scaledown/provider-after.jpg | Bin 0 -> 110207 bytes .github/assets/scaledown/provider-before.jpg | Bin 0 -> 111487 bytes .github/workflows/test-unit.yml | 1 + README.md | 1 + litellm/__init__.py | 10 +- litellm/_lazy_imports_registry.py | 5 + litellm/constants.py | 1 + litellm/llms/scaledown/__init__.py | 3 + litellm/llms/scaledown/chat/__init__.py | 3 + litellm/llms/scaledown/chat/transformation.py | 648 +++++++++ litellm/main.py | 35 + ...odel_prices_and_context_window_backup.json | 46 + .../provider_endpoints_support_backup.json | 18 + .../provider_create_fields.json | 18 + litellm/types/utils.py | 1 + litellm/utils.py | 1 + model_prices_and_context_window.json | 46 + provider_endpoints_support.json | 18 + .../code_coverage_tests/recursive_detector.py | 1 + tests/unit/llms/scaledown/__init__.py | 0 tests/unit/llms/scaledown/chat/__init__.py | 0 .../test_scaledown_chat_transformation.py | 1213 +++++++++++++++++ tests/unit/test_scaledown_model_prices.py | 46 + 23 files changed, 2114 insertions(+), 1 deletion(-) create mode 100644 .github/assets/scaledown/provider-after.jpg create mode 100644 .github/assets/scaledown/provider-before.jpg create mode 100644 litellm/llms/scaledown/__init__.py create mode 100644 litellm/llms/scaledown/chat/__init__.py create mode 100644 litellm/llms/scaledown/chat/transformation.py create mode 100644 tests/unit/llms/scaledown/__init__.py create mode 100644 tests/unit/llms/scaledown/chat/__init__.py create mode 100644 tests/unit/llms/scaledown/chat/test_scaledown_chat_transformation.py create mode 100644 tests/unit/test_scaledown_model_prices.py diff --git a/.github/assets/scaledown/provider-after.jpg b/.github/assets/scaledown/provider-after.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bcf1c59c96fb5d5360d4667c35508e4a75d8f8d7 GIT binary patch literal 110207 zcmeFZ30PCvmOmP_91(F~P(V<@iLqo|yQo zTfem>pC#j)*~vgNlQ;Q9vm*|u}rwr{@KwtL5p?K}7E z-m^z~x3sj(-UIt&_U_*+Exk`+-~NMg^78U~WEBr7$Q?Q$ColI~BT`#|I^S&DwQbuj zIT>jgx&Px!@*cE*=a$}WpsiBJL0k4qZQU;=X$GkQa&D9Q;|2QDCA9^Rar=&)yLL+h z1qk~U`Voc^QaS3A`&SALMc(Z|s+ zxcl(&uH6cXhYl;9IC<*y8BIeYW0P~|O|N`sWo=_?_x;UVU}qOsH+Qdl-afv5F#o`y z;E>R;@Q5crMaRTGeHNFJ`g2-(#xI#!uk#BEi;7F$lvY*a2{pvpcXf?T%`L5MAKN?n z2L`D_!=FcJEcV2g$*JkDGqc>~mDRO%9)Dx=H@&0)J^y_Dn__>b*M2~+E#G{z^_v~P z=_R!#6u7tU|7P2#v%(9+k^Ow)?q^d}kS-Mu#$oIEx$V{MVPf)tv2tw%~guODML z_9d9*3u+v0(#VoHZkGh~LHCse^l@z-b9V}}Dsxi;0%dF*j`W9iB^91)Ifd4mj*?y% z@prLxdq|odH_s?b+N*7 z|C46ci9_r&9eW`I)|J2da;m5&HE{`|@>jCium|#k5c7}s_ELVq26qkXK7aW-<$01j z$o-6g{l%T(FCT!B6$^MiicWfpyE`-GaJ*FPQ&96Raak_w@B!Cf1Kjpax;k~P|Ck#W z6P{o5>`U6vxnA$gM5@zu*IDPRUo8-YgC@zos`n0P$LE#ybr!xS**&^cR$FgV%Zpx|>0gRM@zNiIDK)2}<&?W3GpvJ9(_Lkkdi!(%1Voeay z3M>JAF98)n`D3~m>jd8LZd z5O}G8xWvQGVbtM8g5AY!a!&utZ?IkFwCxBJ@OSw)aJ+(i{bYQvPu`xRz~s1#bdT=F;Hya!cIlfA`+*8^_0pci-exT<8BH z0qxD|X&gJlH4M4pw4&H_T|4Hu1mxZJOBaq9GC-UZD^%1vHlV^Kplw@4`Cs@T)Xqx2 zNt0Md=`DpJBJIdt$7szsw`~YZ-Ic{GH+w zMpgjTPu*s3 zd2+=seg04k1Nb6geT~-m@HZcY#-0R0i@@<$aYx>M%)Bc}0_u~2ZV}VeDoGhY+jw3E?{bYPajIIQ< zdc`HS`CYl1NiOv1E8!;z2<5;TEVMRKSMQL3#xp3({^@G#^7Rr>Z#ud5f^0w8Mgn^5 zDFL;Ye#U%uUXg$u>O^o>L27usx=vYSB2x#w5=C2~JYv=i$F45XFl@C1bI3%)UHW^|pA6_< zX*x_=IoPczIzRc2mEVMD6d@j7vK+%f`KQ{NDAK5l{14m@<_;`|db1WhO4+0QjrWn^ z)H?QD2=lB5lHndYzs70%u-PV*vtcjAB857ynPKS8YwscJp3bZ{fS)nLB~ZyH6* zgdtvW`6|yDaaJwY|9Pf^`;!MzQw+iPQs^0k!? zR>8_mU#x59Nom=kj5$x`9YVSI@$WW!2Qperjww0nAfZ=lJpz|V0S}gl@@xtR47ohf zeRefo)@e!FwFe&u${0@7D`@I$!T_fAd}_b|otJ^5Mr4Q1w&*l7GAc0xL|r6hA`(y8EeE0xGC-`IN`` zUUtJ6=O}J9s}LRB79}tE4j5EYd|*|-+xn+66wdc*e^n;@7r!i|T| z@1kGP_&qiaoZA-K_RZoCDewK^A#IZa%Zw?&j*-JFnnchRMF~g_ZAwGg3?5DcBvq^! zVI#Bs%U%#ONaD=46Lk`hz-XxkO1PEI;SK_WTMr zm>@Pf@^6}|%~!KFSj$G%;d)3anMd=kC8Ss(d7;{v^-tvZqwj4_;oNh)mu@}$3Db0l z*uBy_-F$1E`#gW$aX3woKVxr835q%c=X#CqKK=T~zrN374uy?%en|3?8ycoXGgcE+ z6e5BtIfmITfyPWL_o>y2M1_d4Jo$yBXmgi9_{oJ7tHMQR8XeQ%EChP_1e7%`YR0VQ zz@PISB%ocgpX+yg*ady*aHH1G!%GI?{&bfqK{t3oeEMsKdvnBnljY6_8o8INXx(4%*`bJ)nvF=n zCJ=5bwtiQe;SPSg!mfp=&e|8nUr(goBaD*rMU(aPZ1F~!noJcj4o{B{TYSwCz@=3} zy5CsS1IV4?>UM#}0%aZ0`5A5_YCtQ`mUeaE$~*GLu@_|hJ` zmmWO|g-D+|G$FETVqy1y{zigQQvk_jQOMLEI-`{L(0|Ob7q52sD_Ws}vJbvFZ*mY; zbJ^IZHjx1?LeN9XnP<@}77JTS4SO&6Zk#sBZ{oN zg;$*ER<%EP)V-Cwle3da0q^ObDhnTpl895z?=9!{6}}A^v!ke zm_uDD+OT)@-6jU5B3GB`dghU7GH0n(6fi5J6e+!M^HSeEZd;SqJG`qAoOrja%Ksp~ zQ{`K%jY`J;3P#2fsssKB4Z$+H#Rq{#t(!W)zW=HQIlUYC~86nLHz>H=%#)o{?< zv;3IW*#R*y9I={s_=&o*?2w8v0oMNK^GSW1S}V{djPe>&QK-Uy#kFZB=AN#s!Vj$K zZE4z@jFr*1HQMHi!TYS-b50zMdOh6{G9E-vUwS8YDAhlKGc7Aq{cz4nj zV52aPhboMPJ*Fv_)X(VZSbd?pduQIEYhY%1CW66-v1d0CR;YG?2ETu0g7OS~6m`cW zw3Fpl!4-ZbH_b)jm;27@xAB69w zAjMkB0nkj!SmE6kjsz4Rgptecna}y2d0n8XCDk2ynlK-7HtVZqoeNK{m-w z1*}^{XrcaJ9=ND;zaP@d+gn&Q0%6Xke8LvSs}xj15QwVqybvR3W@h#z{ar}SLfMat zT2J_IxEUYvzQISzxU_DrdL9`Fmi{t;TP%*VwyyQ|0dIh7gq4+}OubHsbG-aoO_H}l zvAK(B%4O^W^Rx4bcE-XktWYa!1K7w)kK3!oDakfk?)1OyI+uI?aifoPZEF>b*ZB@Z zgU~7ChP|o_+$$3{Pb_5IU@p1(!*}|02=>&0g*}z+>JDbxy{&FJBUy9(qz+*MKN=U7 zF_G8wGnpHVe|uY~5^Kk4VE2%CwACzXLqivx5-(C1vJ~zDHqSj1FS9Dzasrz%`w2^X zgts}ZH8tdKIFm({Cn%Zl$;Kg!Z>2Lmc>S4@RqDg`WeF&x;zgp@szI~+T2l1t5qr16 z@@+-gMYQ(t;PaQqZl5m9d3&C90Y$6Ou4cRmU&@^`Kra~~~DTRD~t&cAQc zy>do=V}v{05@K;|NCNVc;<~hDRXh=b^TPrL!)`XE*A1rsU^fjT6)YpDX-IQ=jNd(H z#o-=MG6Q=QqNoCEe)Yt)Brb&&ob(fZVb51Z_xV0r zIBktVSx**sXBH%Go?94kHbS1w5czhi`1(sgvaiMcJ#Xv<$AnH%?!eCWI$!H1yab^E zJ7$M&I=PmR3C;`+pmkyClds?V3D1e%SSX=x53~AEXV|;e94`AQv5uQUzU=O+;D!Uk z6mXbbaKlZQU1E)~PLA14JccIyA=fZV4Rl-CQTnSw&H84}^W+h%7CxELu5RdGh7%Ar z$jA&tAVzipWyk49jtoG73H+?yv3yQ^fn%Nj0P8lVL}-~e8Frf4&+u4?v-VC%cT5Xe z*uac?#P5r0VJE44^^~P>M7h-^I%%Iss5;MVW$P~A+Oze!t~AOSy0Th!(r?{F!6)_8ve;*X=wicOuxVC3%{`g_ zk`hM{nU1Zj{|GrUIS_mXH|=qP-}alB@aD^>Rh=oxz}sS0RtGgpYHeb+^9N+ z*z-P#)Kd+acdi&dS2)Kr;Tn7^1^1_%6&vt9KQ(=T$_r0m=iDc!e-g@Ym75msSYXQ` z2C(%|xXTy0c=@humF-`;r0)NFZwGF*f`+Y^N-({3?26Znj?MQ`%II zNVSy^(jjmzcSqJ zjo=H?*T}bOpPeEmLWuTN-lL_TlQGfnHlqCp)8(A3K;bfHL=+8g3J6D9t;y!{BwTMk02*27D!o zx{)#EU9p?fT1KSZXcJY#CeBb;m@xD#Mj7V+#;e@Uz?%Ll0qrUzfD_W(GDJlJYRFNM z>1&DsLGP(cyf2oI6CZ^?5cl<>E{Qt{obP9sHH|9h>GHqLp&I5txb2<=!s@tcm%bPa)3(N7_Z+VTc+!riiZ!`PpGFVWVm0A?(tL_U;& z_5;RYHd;P&OT(vRrLyHFtwvDh;9n1EG|-B+-Jvv z<3e@`b>39NGCVKi+u9R@U8jx68apw+eS+olR)Kl_o%gqI zHnNk5fM3)6&NwQkcvfNaCoK)#;z?(3#)|GFV+9=~vJqbx5R+^qpnQdT$Mujt8-ULkEKD39{WPHc(7-TZvh1NXE}Fq3BkQrG$qM+O<)EG@1kll1$M-bN>iA^% z(lhj_Q9o5<({03`eCxwo^KAm|4oYdm+rXKS8E74-#kl{V5 zMwSL?FiZYAx3M1qPP8*YRpu=Y7r6YKonyncOqzI9mg>+~uZ}fv(607jFHQh}9tnJ% zq8%9^&kNWfgT-qR1mpdhbWB_Kv?vHF+6D_aobK@lSvTW6!Q>kq%EZ))xK2;}7=Ig6 zUM&HY17Ly`!OpuBaDvYRD6;(LkuiVMdot@dpZ?B;{A->kk}VJ^to-LO zPZ-U+NgCp_*tt-AExp^=&e263^EGj_zfpS^T#K=&?Z^&wQ6uA}A=U9P7-U|b69X5#v=S2esD zq<|%oOXsUoRee_S+hn%X%G_kD7V45PMf|`f6V+o;=SmuCP3+)%9_nLvg0q*}E7K@- zq>QQZq2-19IG>#S6z@1=jP8li1!7f2pr;O}jXfM|7`0__0$f{wRzAF8md?GqUwuMELq9<%Oe?+ zv@WLRY6*E?*lPexY_TwSJ1{}29q=KGslX;PiK{|z1B0dL#4CRJ^8)9I--) zky(Gx6zrUQ@$)U^3<`tZy>9Av2?)dW7;lR&Gq-#*8=qZ^EbtrA@CK(M7Yp3>*I*Gy zc17J@(E7`JvhI_KQO^@~UAxa0yt{|X_slykuGo9+$`@ezi#xM1ffk z2t$id*4%~Ir}zfnM&2ek?QKK2oTt6&=P$_7a;ku7B7la}XS0^(Ap&QiW}gPI+;X>m z*NbnpW$}DaC+`# z}mxC@6Q1E!!l6YzkGnrGQqhh3Op-vu!_P9i-Lt?j!Qjjl~O5Fo!+oaVMJ3 zZ8#Tt-r&7yOSecTjyY0i3QzM?^Fh1aPgmraKK{EX)?SZR)g4vb)X$wp)U{~F^8^jW ztl+j(!^jM{?vl0pv|+`g^@x*s@@YwNLNEfZx{%!VhUF3%AYO>@mRo$^%3;Q8*PnUi z4l{jtsB9s%Gbu6viB(<58hIUe@jnAVLGsty1B)6dt1T`P&=G9fbz0Tgv)^AgF$-CA z2Q?0%WDEJx`3MiYx70nBv$W;0^8m0(dWUwu%RgufaSOKU3mT*(4h>vxEe4+ z%fnYDZ)1Ae#8>h=Y9*k&Bw;ML>*AcN`O})b241Q35#3J56t=y`IUU-ha3=Rq+SLlt zRNELbBID1+ZPFe{K$dD7dkd~315;=B#wa73V;n~#|QL^*OZtS zLBPTkOecS;aZigTzw$jTnrs-=(ng4viQ5a8%Ppq?82DJaiYJz@7T(is3%m>o$!$-Q@0%+|urd9;_YQkI89lg*R0 z>=c$72$a}h`@jE27v)l$Ht)JZT+4G{>sqh7A0XF*mnVG$U7ZOyQbVAsCf`dQunoaF z2OBer-z&O2XhVvyFWo!}IL$cHU9Hwmo?FJEkTjAZ11bLV_e;{85Th-ZkAMO((rA^h zlwB5Tex7?d^A6h+@;!8Lb)pEcPFy+D7x( zZTmn>{q(FT!J{)ZFB4NYn(Hr;lS&b*^cOx}+%a&Uru>dP@BoOqjd*Wh!a9>O$>0sj zh#Z|F?MF+*BCrscf{AJ2+m?pa^^$=m@m$&{CHlV8#Y0)e(T0eCMdfZA+qZ*yM-%d6 zBD4Lzq!=OVOS|f~#s%b$GiB%YjeNkooFVqQsnA*ieO2Oj2p(ljNg&vL1fY&n3`G)QVgbwoR6-s>#e)C=%ch^{dtWxDg`W*m-WZ0VdLa2qy;u zo6(h{E)T&yN>Zk1-hN+VyhqD%yr199p2z-teTPA@(413~rf%=e%2EAvMcLYOtntuD z8hKfW<+OOYwb>|E@zOu#9!&lEcmrJ(0d}s=Mk*LE^2RdQA?zT z1mqw2Gd|&7I2k!+fcUhONk*<`0W<4~npFLHj!UDvb3$=*R?g6`yxeOQk>6n8^asH@ zO*T`SBF2C|yTKw3)P~vpwIBWrXOVn&vtKX0F5bOuJsP_B%y^N`AE2;~^w$Fmx&8o# zMy6Bd=ko9f`ZR_@oMiB@?s1^@yrwMhV8l;1VIe6tC73o^dq5e{QkOY~iJaU(`otw3y@My5 zx-?)&PlptF*q>wFwHyN%&!QFkg0XX23pO@YgL?O~gK^#6We@EFm}3c<4~s=K+Up&pzfjlpH+*p1F*q5#jZZ`{-+M8ZN#W)w`LCsTcBQ?oj`i_L`F4*$8_O@Ww2?X9~x#K*`dU)I$vfyZZVydhLU2NljU|oSkdnMeZjG z%i^^a*o^NUm@b#)^Iao`8swy&kIDCCLm>BO6m)Q!8(vPvm7;tk57kYo8ID6aihicE zKUJ0$RdN-ZDV(S9B;{F8CD#SF!{9V>kIcZ8=_YAIc%2b)DT+Z|Js}5bv9j8o?;jT> zcU|i5e9&v`GdD2CC!9Azs)*d{+d2U7cE3=HQxO*o0ak$PiM(#=hw7c??yo>+J z>Kmj4G{LZ(!(%qj!t(x9u_T~4m;LQyD%nk*%;UMazfekh#D?ykA#27Anepuj4b3KJ zZEMGHD$kA!$Fh?jYT5k$rl>mT#yV0k`=b_2RkgYgxfV}~5{JAABDzRrTh_aJJ@*oC zRT^q=5(H6J=9vJS1YX;!K&=)JT>mDW$z1^`9uOd&ddWSAn~^e_G@c6B7P5yBRui(Y zyV@)uTsaV23OfBd4)l@Idw*@Gk-VSt86-~JeW4ko-K~1qraOr>`uTO!rRst4fwVhr z-jzgwq2j?G2eiY$md2Lx{!@gn|26>L^&ALuGj~|Oaxxza13doJF7<09Kl912eeJ|0 zJ@Ez6kbGa?@0=ca3|tQh(;D=G-ZPpVigd)Hh zz$oCHF3MZ}UV}nYpscLPiiGY2E6^=*FR+ctE5R(W`$VtcfaSanP-$AQLD#oI7o?#Z zo3UcCjvCGQFEzm8zQ5GS|4WTqztvFwQ;k#W(Z9dE@8aKxOZyvfFaHPPV(^GRDVX=W zxI`<^f9d7_=i>ek)c|iBjgf}kj+(T-b5S0N#N~FdoY^OR63EcA7!nXU^853*fM?bW zp8BQxU`N09i*{0|00)WwR`ai(`~BscQ%leMC%yca&xZNE@&9~W{QtAYufu1{8V?>1 zFC(R!3%fL1{$Dn*{${TI4-MXbz2m>R_Wi3r{%BeLZ(XV#|H>Bq*?Ixy;6Jfm|GWmU z8TmKcsrJuo=l{wz@{b%}`3|8KH;ym+fkzu25f7W2c@YT=Vo#R!{x($%&BuL8Tdc1$ z{ge4ITzR@P0_hxUo7-2kQ8wl4_hq9-NOw0DFV2Q+9pLzVC_lyCk+g_95%n5odM(8$ zC)6!mfkv|vK21M29F$^ra)^d~hl%B5Ph#RtOhcP;T$)Wb)kcD~&WKwLKQE@cE*ANK zlR7h9rkr7lQ*_Z%di?I`qAyO#o+en~eJVn%;1R}j;H=>oRW`>45xT+KW9(thjym~J zhVfE%C=b2h#>KQG?F-YNXmU19OnOD1btgdr6io-Nz;`#6bPA?omyDMd*NZz^x+2D` z4$FUuV7Br&jiU*U_Jq4+6~x0`+Q|B^0r%XDqjHS_cX=#nN&-SI4vG)*sfuwG_@|WW zq5d=YGe$UT*eQ1Beez8k9vYc>PrS5QcmAGATj_7Io@HSYhs9?&dQXiyp?lgzg$3az zJ8a<@=?T>;=kd$Vffk{h83r=S5k>+ZXJ}7|4QB`B+>T zDLZrwfq&;3sQSoC?#AigHi-Z1*2Dp5(vwBXTsG#bk$))@p!i$NiQf^$PcEuNCv5`C zlt09Lttp?Xg{!S~1AKJ=MC?OAx^MjmwEW_aw7Lx>8`F$c#>DZmHU_^ut3MJ5_mwYz zt834kTp|JtUPh-%JdZM+6J>Y%Q|dTE-G;NU9xLadldy8zF6d-bx52fp{%R9}RaU8P z^G7zyhRu|Ni%)Y^b6D60-A|pD-^PTyuQQ6^BPSCIfHV=P?gcCm$H{tE5v!`fwZOjQ z8#H;Uklqf^h8t_La=NJ^((UjfhrYJmrBy6UNK3Yewx}oG85XkYmSh4ZE^B)YoDdiZ zlcTC=<_n2YlsQwPW#rVG0P?P|h)9U)p_7BF)BR;JIon#IS6^IRZz|B&7flw2J+dD^ zB1n3?>A)DcA{U3$NX~L(yb;`LAtM)mW6dQ+i_h{O&w1kU?-~eH+?tN@;^LVFd?eN^ z>|V6|<0eBUmwR{=ji4}q0M2xbaDPFVOXJxvpVrZYQ0PrHX@~xRsMf~h* zDRc8f&Ix;JDvngp6sKUyU8ALRGMGk9vZ!VQK9fl!F4Cob;96SQ+_#XX;*rLg6Xk+% zAZTuaUg6ybN^Yb#JRW|pbfLgPG4b5wS^h)4lP&HJOuf`ELgv_pR&Kv|TNvDSaJBOz z^v2nlP&NOz_;j(I&Vuq(N1HN0-ZfHu>i!+@B9s+d+VViy%_|&Hch2*yrqMQ|?n#p4 zm$Dx=GH*FUfKjxi-0UJhX~IPOevn|FbKJpr36k10FycY8tuoNEQ==(Zr}7JuM4rq# zW<&$ri8Gn_1~;Qr#uASd;&A~TpTYVzmBhUO*y(+RL${9U=;*tFIoack@KLerXBDJa z12D;AGTHs$#qfrt)(iLe6ShwTm)l(W)-Jd^7`7D8xY)_mExc&$j5@VAg<%Lgu%ZP@ zNFJ-_>6m`jKvSLgRbXLzzJJ}^*X}zq+1efCpx&~vJE=IkRag}zByBvxs6Mwm3sNCp8h7%!fYDO<0 zXbwK3j<3!6K2P6vm`~JP6t6S0jRxk#N|jtCR~G>7;cRS31P(bL>Gkm0o#V!@s}FgEpn68D%H~8%neOB1tPQktHajK7zgFKKE{w)*}yR%}8&nyn>R;{7D#+ zxKPUg!Z%?p?Sq@R5Rb2)LwYB|8K%iC9W0nSisnaF_hYJ?-Nxw2v7!oUeNIX0q?W|+skR;SnOt5kqv&-o^@Y^DK&~&8xPL}z-?zr`1)o25 z1erB_$GpbbrC{tA77cKSvVR5eYR@!w(qpkI*R!}}3}ROTuxeq}l~B4H)NYSSwu?VL zjuM_0ag%^~Qm63dxNYieM4%`^`xL@hY%4Rkvk>faR7iiwG#T5d8>qkBe=e3AK(3pP zCy3Ooav={LW&C(fGX~Aw8af&os(0}iEd%Q^CeZIhL^064XgAzwa&1ANb$e`9QRaam$;BUQfZ6?g{6@fW^BW%BE+5cl{5&@HSx~p{ zt*WfHYxo17H}2zpcu{6qzXI@`uIA|jJW!@i5P&5o0Vxb2;uBTWfK`Ua4Px)Tq5DXk zd73;_*DBHWOMKUTVZZJWLo~Y@34|e)REx~75ktCz)t-JIMLzGLNmq9rM)J^LXj@iW z{CMtqSg8(QwkQJat94%lVE?t9hN!4w$Yz;*<}T<+3icn+S~ab`Lq zwiUC_4OXLctB4UiL&#>RBEBDX$eJ~>zf;>)Q@NO&nU*ohz@xf(RuQs|c?xr?XL>6p zhvuh(##N*XS)XdSl2NI6zGg$B9SQF$P^L7H*0^Y=j8u7#h|+wz)?hw(e;>?CxOUVDcSR z&8}*I{AosG#qdR&a{F0i8o1C?9*C>cD#FahjiO7Ha~|5apArw(Qk5MOH?6DE9r1=Q zKK6=+yq(lUE08APh2E*YwQha5SD-iUwq0=eH(eetIKE)^wv(T1OQ;Y347JrRT10x2YZKRaj@4jl7U}2fv~nAwA6|8Y$pELw(?x75LzU1hJkHr7)aPu^e6_?UbMbRUI-ukGIo8+$pnHNR@rMP=94xK_9+b*_u+H)CDs! z`GsjbC3q*+TpQi}?U@z*@N3FIWOGC45uWcK5r7jtD~BDh9U}US-d5L+udyA|?|kP9 zAX@tbgjRR-Ck12L-n_z5|H%0PSA?%q%qRN1eXvPJ)rq9`;N=6BX=Dg9q@jAt^WCVf z^NzVM@vnVdiFVe`WhrYEX981*WkDkL%piToIFjHW?CL zD0TC9jzDdij+VAzibQqiQIl4as1^YU%pc0enk^^#}ot7Y&4;bralC(}O9o5r0k z`1W<&Y0y8rOMyN9vMi9^Q;v{;UQ$c37zya8pki~1`CC4Ug5_Mc_+2}Y97Q8#vC1&R zCZfR9-zZ6;MmK<_H81`%2j%~}&+D0NbOTlbf`=g+BXqd^Mn}+g!fpqz19ouap2z|} zD*2|mf)(U_^k{cd`i~cm$8CsZ08U&~c}qC5j9={1cXU_>FLQ06<*bch!pue`G(Aug zt6!GVT1E*bBhx~Ldx0p{?X--fvHjYAG4ge8Ax3S22RAr8_i= z<)y!^H{upw<3hUg-P~@s72byUD>U7Bmo;{}gLb4s6KnS_--ofHDI}qRjNh4CF1v6- z7{?`c$uK^$;sC5$w>WhD+*OkYb!EBG%oP+nltI*9LrPgW@ecJtMwGC0?I@oumF95XVhLjZ$6Latd zLz-DIYNnD78`q2_g?5!P8}U-13;Te$2H?tm20^% zMd5XS6R3dhlNGKy0(s7OAT)E~AduY5^i2|nSBO?@MgY1&E)Y;b^{TYw{rPeJ?+up{ ze=pGYm!@}|_5HoXfA(srF1J4mX7MKxuzway?N1V1{FiwDLC5{;>-}r&{cGm_yKCCN zrpCXf#=oY<|JJEdY8C7~y4U&5pH02Zah)tevpo^n)j?^-OBDlK8rQ!%C`T$50I20? ze1pcUR!Ug;dIaRx;Z*G_f%>h_yRb{k!#`P-hNaabVNn+fYbv;oH)0X#3EHU`pD+){ zaBJ?(9Apz^ zuh2r#{w3_e4ExKQU$6%C(SgK31A`+ZfGzZLuX`J7>6y@}9HtbTh`I!%okW1&*W>%~ z4i|8UF;?i#sZ0Q2eo?;AfhB})LuMl)a^$Y>h(-5#Ds#d0n~KltWs2?Jgk?J>E?tHD zDZ^66rWAgo6PE{S>!W2{(Wm)lC1D{wtg&aWW|o<5UvpK4f&|Yd)bw(63jE8M3wB45 zL*#_lrQ@icC{D3>e@_f^`xJq5vA(u9plp$VmDW1vJ~36Day~PyW+P!*9|%8?DKXW0 zLOu4JN2c)G++h4JyQ}6vUDtB|8 zRUKI1KL@WlpGoGWeqvBe(mM#roVIcAp|@{mveuvBVEgENp9w2U)@(=Xr(S#b!2Ks!MpCH5SZNiDQPlIXYRr-?;KVSAiJKNg4r_?$;Tfty*&dTfZJmhX`GrC_?Kb)p z!plkt7;@LMwc1x}4d>`B`?q;L*rVOPOzgldF9LMsskA~jzbupzI>&cgbXwh07O^J5 zE73?iTPmVatls|~3;VmHCVod{{@N*a|67l_cgL~6+VYnx0l8P0;q#WO9xj34qd)@% ze>^*=x(qYX$`rq=5I=D@%JBk7yMRx!){R+J+L>(hoA)8V|JMvHrNf7U#j`+nyb*(k zMfy&B7dksbI#+Z+zva`(xwI;|ESO<9bL%ytmxemNw%n!X ztC4Pomh&WlLu#UgHw{>@3}R|yv@N{!i2@G~0pSBmTSQ#bi-m5%do=h=3j$j#w76)3 zZ&CZi5f6a{t(7)r+hq4(LDxG_2p|lFnUw54;PTxAq&j;MO4++g*dH7j6aih4Kvbnh10ygX_YfyCoZkK z3m}5szC{4TP0J+nfqvUbDgZW#mKKNOgqKCH$;bImnxWO{)vSh-Ou{{1Y442c z>6i&SsyQjD^5gJ}MdxVDFsk!i2#{WF|FdlE>55cs3F!XPK1WWH%WyF#CiWFS?+W*} zc;Ao?p-0Z+^Gv$IcctAw1l$9|)M-ukAGCHcO?dFU0m}GG_uK{!sfm2Fw_X}n9`?kE z%M%{G0jqIuowF^AZVF7f#>emLhXzq3(w z-!O)Cgj#{AQ5X}mXWY#lAgRZUZRW640!$#u+9gs-<}EqJh1ajc5iv;0`rbW``G zAm)Dm<=IYT2ynIxJ}R9F*9zC{Bn9fuYVQ~tbor=iC?YKx<|QbqN&~6YF+uj7W*$+d zv*&^6q3vp9&Rw&QLk_fvZncb4>A01Ef6oJe1Oz2DFC%sK+?5ZRZY1K8L{GnLbDp~T_)eS zr*+?`e^yE5@~g~DR5}DF=%$0g9!mx&x^ukLTDLr>Cl&+mdCF8I>kC07j%W5AuW^hL zEGxQYJvAzb>}7{Ml@VO$c_Yn`EpQAjI+z?7KJ1k9ZF9T+Cz3fSyC}z_9HSkO%(`GC#}% z%Dur?THXjIWJN}~WoGbx*uWT?Wh@y42!QDk`Uq_vR@?-Ys=qm*w|7Rtq%I(do$$a2 zCiG=vlSZ%XVMd%cPe+t7HJgZ&8{kj_r;Z&XgD&k$0*|n%&T*5+!r(H#S`+!;V73&x;oH=V*lh;g%`&PJjh^#te>YR;aTkiAp z+bqfsEYW`9dD{bvUiu_3({Jpou}Tghau&pzx2v`;b!f2}>g%ub-kV_dYa2rM&U$Ka zV{lySiN-qPv^R4pc6(C8zpb{LvZZ{jnFTCJDU;9Z3~g2;nmk3FVlnrR{C04_UB{4`+Zp+o`R4(CMzDi`KLJOKf_7?YtI34vnqN3vm1Q~?rX)h zNdi)#oD)Lryp39C^2Ljj5ISc3mej1AON<26^n-8*fUD*Jv|DB?kOt@n%pw+F764bx z0E~x{AOST~mp{rrd|huMR*oIF6v-{fip3@p&_NraiR^pK<{6rrNQe}lOmvf4y!Bsf zv9I&!g^oB5lOLe=BXE4$O}wRm&lEbSaYikHS@X5n55wb9_*v)g?Re+psVo7>G;!3# zaUrn)9~k)US^nQkqWvRJQ>zKXszq-Fh$Yv0C7`7V4A&6z56VrxP@S|2L;kJgNq;AX zO4ZiUk_wrB{1AQrC`tOo$TWEtC7?glx~U#v zhCUPl85oalOujf$_-A15H*oGc6VsLe*lpmfrlJxgWukTB-xxGXhE7(0cZz^ZE}Wu- z*U@sT>(@pQFbDk85jC^)ivwTk1!u+cDSC@k%7_m`+zpC2BHtEf#tw*{$PB<4ZY&~# z?Twt@^{;8!Ap*vo1!cV-OMyrmG(%K6@Auytn}53t`ETL&TBdDca{%?91JK5=M)__- zmfQ(|NPl}6psv4scJNjJ&YXM$$Q>y_iqZ=T-bz3&8aXs;38)PX6{Y{E-5Rg7&52;+ zAN7L-bZDs@Pl;7xw4@CVWgJsa@O`x2t~w*{mPU!Lshr9Lk_UY+0fh0T{Spw%t^~uK z_0D1V_9%|rTRAAQH|e4FvHgCpsk?sggGa_D@OPksYQy-ktP_tRuY2{i5_U&jo!BalTwf zBf;4WT1m%i5h(EwYnHO=%g{}xMzKE`!jGJ4p%o>wAv>UC?%rhfImD|J4>w&puNSYc zWutMQqB|s>9j=*!%GF4!{=~A&OSOOHvaGn5vX*oFkvACxG*9+$*AmHm9CtiABeN!@{y` zX$HGnOR;;jNRwazft6RCF>hJnueKL;0?w6fCaVt^=9o3RD6O^qcu!#FZWnlDFade$ z7wlB{eX@sPWX+-vGA)bD_`leD^QflJeczwf-P($X6Hpm<1*Mg#DuV-(ss%(E(JEmG zkZlR|L&BPrl!0_)PEDr(I@}QQTA=(f9(-u5q{LD1GKav!WzK&7Y_k zaf{NA@sUGKhRpp-5WV#$XH2Ymxs&%Pb+(2fL@(b}oBF&0q8HN*%LTcjVj|PuXUDwmUVP_vST%<5bWw0(A|@lw%P0uTeYv^fxOt5i5YXGuuYK`E z^F7O{XL+={>-yrM?OeOr^YsZ|_YitTSQD-7fY@Duqr+UF=%599t zcFZFbH%)f>NQD^(dR&fA;yh;|2bLTY$bIQ=eQ(lRTAN{T$COi(gWef_1yMpG5N+H- zKLyOHizJ*qDoh?Eo2Sfn_*@Skgw|ZtAv3fM#oj3bJ>OZr3aav6MBUwWX<+PdWBq)f z5M&#C+r1vg4Bxfix*g<6_gscCQZ~Ng^{-8$1!xhP2)w=ycgD_Cq_E)yT}FB2V5g(H zEc3dra+=AwaxOSXy79>3E_rz7Sy;~53}!5=kcx>%$qB-<5vp(15AB79*z_KB@GJ#9 zp643-_7r80sopq8KlKMd#|SMf7)l-DGQ&7qq{X~^M_6dKpN|A5j3XM$-q^m)4n=Ih zV-5al=70-RKdRFl7Y6zTVdaQ%)xJcNI*f~+FRK2xjxxsBhrx75ryk|9EYeRVag`2aCTM7mWdtX@&dl>?Mwj zQed{-6PDs-Qv^9qNY#8oA9@d&GVd?xSE4-zjL!xllD?8|G)@ z7xrdI5X6zY03V4*5+7+#o{jj^i_zNEF#ejjTzvH**Xy)1RS7U={&s(Mi5+MCsMdWh zRRKinFDJX(DaosNBXUni-STcHGw$9ujns)~>vUR725`ISxL%&>Y^ZeS3bIV0?W?#| z16QPJR=sva`p=g~Mg@;{Z~I5ru(GusJ!D8Q)Nry>VEC%E2 zF^>&U08&L@D1`}aCHna1ErKrG?iS;9+ zs4e(Cn#?8%KHocTk377UMlZzRCSEUO!nQ1YG$0hI_qQshzc@E)j;~pLLr1^9+~+2>OpOUN>f2r=)&rUz zEy!{lcZsthe`C|F%y-RkMA&+DsJYwur-Sn_7HNZPCTnKct9zS}7Mry{uxHnBl0(7vLtLeI=BYf4En7(k zT!3|Y;DxlhakpB)NpUz65&Sd8^Kdsy5XN(TyYq?S`ZT{J8E=nWn6dExWzzs}O&u49 zHHIK;`KaJf7!1sn*aQbhkKjjnL9VbcE}ba3?my#@I~bt{j}p^szs*T2EnVgk9dgI@{FZ(z z6CON{&r-a%X=y{n#wVd0eKiB1Trs`nw39o16Qx4fXoZh@&rf`rVwnveUYnIOTA5*? zfN2q_trN!G7~sr8!jKO+-uD+a?*XI=wLszjlilbwYPXfS)g@vc=ESO!q+aS9Wgw~) zKP^(K?P)FCWK!Toj4|r1IFJ?3N@k+zn)g~!*RGK#Z0Wx|9Hd9hp=YYyj}6eX^9q!M z-|i`EYVNoPw0Gjwmj3Xjq}R5SgSj8d8e^kMW5fLR<-w6SVUL=tm6MhwzPg39d}5X_ zjHTh4lgbRTeTJYz^gIkf7qFTye@(JR&55%K@t}aZ0}6lW=a>IoC=6g$WQ4e%EXqnV zmk!}Na@{nyhUjp~b>^t)qQOw$s2V6j!#}=kb=Ft5hxpr~lYgH2EsWzxYLA$r#e=Np zMeD7?*O@>_=eEr}xdz(!lEbgJn36l?liEN`_3Ec&J>!DC zJx3>j47r0^l+v0J63+cjuJ%`IB`gr4%m!6zsG@q-l`=}WFnj04^1+9otXp!%9)=<9dBLZW%1#lzKAn?O0c@2EO zETw0^(tV1RKFFuVYK4CYLHo#qDM*aqt4Rl);2W)p!=6`LbCzb};n5xz8Eu8EX((bW zmSFEanWA)y-et8UnL*MglODby^i zC=4Zx^_k-WFJI{O8XeF)sX_9La9st=1+wn*V}D=hJ}~1N#u$ zb+^Cxx2>wQZFm>!neZXbvofMe*wcQ{U(B#SS)NzSJ#`PZn_-e4mzZYT`GrSOm<~N~uhO1$PNFs5=QU>kI zaE4SZ8g68e6u11xR;e{Of3m&f)hV7M@99xLbGdmqFS1dr9Ab%4__4!tE|@ZAdhW(i znZv?khGo?|yv3{c6dgs*OZio*%x#e@wh@aVvy1immpGTW=LXlrnj@f0T{g zNik{mh==(JRUKw2nDBIaq-p8lRc~04gWs1ZvT_d99R1y`kqMviZLgKwig%C$41;O` zR^Ev}1!zvcbU~$(NA{?rYvEBSE;2dOHE__#1DHB;b0uqA*G~Bz{^bdrQLZn?PGVzk zlqgQv7ItY4Wmn{AP%dX5=|urdcj_FMY#x;>vD>Vff7s^rSy_j{H2S{1nIZN;0I{3J z-T$uLG&Phz^sL*DN#RR>N6uz>pMHe}}%H^5nc zJC9k4j5nkW+9Pk-50L-RLVcORns!)VQ7yaec7dXIO+mfF{o**|H1cQ`+GM_<%0H>- zEW9$p`Nb6M6uS@~ICe}NDJD!4g=+~2paxh4544I``-9?>jjT73UzIVM{!*4EgQOEo z&`d3CO{BoFCVESeWnG%LmLuNqi)Wj>sz{#tGei`?!<|h%rV$ zF$2opp#Lr)@RNObS(^6$Oay$uaOtLT-!CNNzrfP`#|xac@p-Wr^xCTW?VYBt!NHI0 z$Hmq_rCuUR7u`-<@!*Pd<$$o3ztmpg661t;hdIzdR1{X_H~=6WK(cNkg{yVb6E>B! z)<*j#Shh&_fTFJ_A*f?Eq;dRqbD?39U02Hiht4`B2oT>rd2B1Qc9`lNS1;!MI#~;R zC$q;z)MtvV=&Ggmtc>z!h+XhE{!@xGi6bY)I z*}~;zS0D-|V={aN_6k=K%S4C>LhkjlpDg;zSjh3A{+4|whAQuF`h|F-gez&JpWmLr zh)>qnm&{nD4@181NbREe2lkdGBGIvEgRq5(`=sjp4DyS0 zUK`_DEz|;SSeJH)h;$lgbQ&PZQ3(1Cs@Ko=B1%4CoSar04vfIsU(Kh?zGN_4$KOU2 zM0?mf?On||R9F!P+hWSCuC4x@A~bWp*Wgm3PXFx>t29IY8)yqEVrV4xPLq8&#g)(cH~kNB+=|FuthWZa;2^v^tG(F}7CF zhfhzkJ{f5kV=(3(TH{Q(TmR+t7EqimK?P2g^LLa?GEYbP`}~48NA8{S>$gmyl9C*! zzG#^$&s|}iR0$K+U$P{5bx%BJS&whqsm?$BLn|bF3`mG%-{Vc&(T<@Vu{oE5N>E1;fab0-)t_e}`pjbrUJ&>?R2@Sr6RR#m z#3P$^2(>5HI-xMf>RoV-D{pP)zeYEKRx-8-AG8k#((g zoJ>7r-h`|SMK{MoFheL3K%N94IT7E04d6pD-~U7@{4an1Kdun`579TW;Y(0em}NxR zo(EX?KV2#!{^i1(QQcGd2~{0S;IB20w+&Ayc!)Pyb~JQ-#+V4X z#Qn_B`|+j0!!7*bH448j(d#p;W{Drll5I86bLVroU6*O$HzE9tiol_A<#15$?!`OA zY?_V7mASuXJ#f}@N?vvFMYm*ASINkfp*ffJ^Gl{h51_}Q_jPxm*>1${0J$T!o8wZe=qz+2#w zXBRO@;}h^cCsk9XL)aSo*C<{9<{@4mTK!J2JCpbA{&4AH&XM;SErmL>Hu8$oMVI^v zqEC=0h4K?%cqBebBLv7fGu zblq9rH&PRRK`%=@s(w=Lr#~6TCz-qP2*A3PqbO))z}X7KnRBD6F`N*iUNW4q=eN6p z1QKxBE{lQ}=)#Yda};qmnnrDJuT(TmYPzaa^t##NBXDfoD)3dKk5TyimNGWp1X&Pd zpMxYBE!rM!8=oJSh;QG_Cg03FCw&iNc!8c?&+^wS7)%(_7gWWTfy!Zmq_VvDV#BP9g92A< zN|zZ%7=EXZH*C#vk6$T2&g;VU$2yhlS`9@UF9+Elr4Yym^iZn(nPX-LuuA1wSG4YH zb>rmVvu-3PWG=6lho$n!>lX+ z+UrIAxHG7xsBD|Ma$j#2*W3Se$LPJ{Pr#PD?BAN(YltU`rmj(SoNi=Ls8+tBHhC3l zv~QyZe9k2pT4FZ%9!kvm>MI1vYqb1?***tYyFf+$9Jr>3Dxz|4j z<}{5EOAlb{qbj6m{Oe4s4qI783ji%{h49SX1_p$Q^RV*2pZ3zq zy4+k0(nd3bT^Fy6nse6MY#FC5@}=rp$PwDbxvk#Isz7p z4(2E5P}1j%mR;e#Xve;dFJO@MBtZZ%1bA_g!!*r~pnkOF&p#@{X|KXL`><|{u(0O9 zbFkA+Cu*B6(v&=;*9A_DiRlvXSMWr2*1X4Ela9Z~;lBqV2Kk{l6`g=(Imx`EcYZz7 zKG{XSVD>pS$>;n;>}m2%CLC@gb`dNI9CAX#MLA|CHzCLBr}S8HESsnxtl_qaXE)E; zZX@vG4-@wAxwe*Mf5*(zWPcHEq+_l5-iN$sLdT@|sz$QeuN-+#a1sD6{L{$xjgm5G zu#R{lZVVBSl+TB2{5_~dw0ezv{bkqRWF6Yi+ml&o>%#)$wg=t$+wa-)r88mPzN^+# z6(DFT9wHlEU?QYqfYYf_phbbb4@4!}NyIMs)Y5@)!{p-njqjhm>9@$PXGcIf+VC(m z9r0~l10&;$Y+TP42(%6UYs_Joig6NrcrITn41N<}@it+f+kaWP2y!jU=(DeF9&hfh z9T0>ZZ?XkcIBYF=;7tKp{x3|z2wpMVF8wSzJu+2)+vo~4vmi8U4;9F+*fc%fG7Y& zs>C@z1B?``<;V`(w1DTqzdJ*x+bRvTW2e)HJ)#&8^sUn08*RG6H?`-ZVA;e?R9h+f zG-07cuoW5T{@lM-<=MbU`usg<>S!Z;aQMQ@(>4rhGAyFB^-Uv4bJ{@e4GavcAP$M_ zgIk?_MJh`vzb{o|ig@=dBKZ8oOdA8*sh1IwU4ga@4OI@(1_7Xrr*fD}=u5V>XMG%* zgI6h-4PmPHs4E=T&fMe|3#TfsDgN z=g83)pMz}-W@qvkUcyf<;`z#^Huf8 zA?ZV8um@_f9y(w9oS>`s%5u_vxirFgR|A=ma64NZu32THNG%;>X{UPzu=<74#-x2Q z#3zt%;b%w0I$=hS%pbwL8k#YlNFyf|V6J8*ku+i-m;LZz_;daCk)NYNXk+%vhin}_ zyKSC>Os~xFz8T@N)@LR_OB~rs(7Z_7)mSuu+lrXx!>=kxq45||Df&R#J6s>CeU zq3*B4hFb+X<{l^dm8^ouS~Z_UjV#Ur)fouHC-1hSh@~SHk@`%7342~v9!fP>l{2#X zf+Zgf&b^(to^&ue^=tWD-%s!W<-K=;y=||Yo$fkj?dEI{ z`plz3v*H#Zj|d$t%-pifJ9EyqX34>qIQ5if$)28DchNISzq(Jvsq6YMWR zI+n~mCscPn^Y1dk4I2*(9wE8WSs1z12{nfwwDsYtFYu?{8EC$wkBPFrHS9>^^6iRS zjtlyYhl0H`fSsLrZjiqDm`p=UCmn*Pm91oZjtud@j^TaRDRq4r4yF6mt(s?Tjy69Y z;E3l>(0*m{fj~HEr>pU7gNV!OA{NF7jfA$z3z1IkEB7vy`iKqM?@{ScG$=rb%|Gpe zn5pAm|9uScZo4_Z*Z|}eKSuq?SsJ-?zxR-HOQycx(VSr-(ruWwiIhhnPB0&oW!}A4 z;;^jI=BzS~i`tGz^Sd`94xmpoMF>!g&p+4 zFun?otdJ3Y+L^k_@GvEogOw9sF&dT&eN;}nue?2O({0rTk^ox}t0YMtA+19|icCNi zHw8O9QP&x8a&4z<>Vy5vvk6?7aw0l8%Ru2rUep|t8Kj%Hys92WQEd{qz2neF!Rn!m zutvu@7?*Xd`Ejk#c4wf%eMQKPiNc_+}!Do$9$EGi>`))g?gYyxN*8Vv%zU34nUM6RK!^M$&;#GHw>jFce;$?TnSvTN%O5n$ z1s0LYTO6%8d%l%Z&bc)jGZp*r4=wi6e*fMm0}U6&hyPmx`bj!hhg_wCMkkzLlD(e8F_lNiMx2AL1&!{W=D1W3*`WU>d!xNKbUKVO5qW@UWfkqWdLwz3R~2TQwe^zV?$i zavSS1fU*oSEAnQg&QnUmYZtUlA=1IFQ9qnp$3!!0=Nqh-Ywo3#$$_HJL~>hwx3Dd5 z7M(V)v7zSG;J(iC=1^jdSbO@Sqao?Dh1L0bVL5&M-e{sGK1n&*#a>B@?utassQqi5j3nN`6 zx5ih|j>8*9DXoR&;G&z^y`%MW8gU|7FF-Va-PwuOUUJc9o?8!g(9^ZdPRPv;SVR~knj zIEI!4#5sG-BvqBq_DcDRQHO)0EA#D{tDQ>E%q6em8XEEaJFk7sA|NC{`QV1|J5->U}Ueu|@lXB=8%`d>X5gtGT23 z61&r1ftZEDJ2hW66nL?lc`3A_uMMKatk?qOfG{4e$Rkm7HXbjTK9W|?=yvkYKL11O zHPcy|*~+b~FIATY&!9h#+{YD)d7$!aEk12pSABD=_wZB`YITR zqT|tfNWn*P#JT-BuA#p38oQY-*fY&>{Ocm8Zoa`xU`GAVwf?lU^}wB;_n{ZIqAY%{ zvvppyDDvvCEL2N6=<|_VL1_Wm2FEv?9{iyd#XGB;8gy_xwnTB|FC3L!d3`!ESj5`r zSW+3eueYvcE#MgC*T!RwtKqzPE^Bo%Su;{{s8n%smJILqX_<6dLxi`d6fwUehWOcj z8S3Zcv>%U3Mv!e_%U!P?Hc)c5`7fytjG6&`61W8yZ@~3FFVb zAR9IU!&gAL@nA#T0!*WBwb_mFof*_DUEVwJ{N4*6GQOKMWQ@LPMp{i8ZVJRj| zf0Y0+tX@)Vq@DY_>rhfsyfP{R#yt1xXv%nRkhQepOi*e0((YYba z5R!cqq*?A)Zy!dR-H2dOAzdY@3AX9w3SH4^r8?Isjf}fS@yCL$exi^y0(*)l&LG6m z3-f^EL88aTGF{Xx>!{`@=Wq__8NLs!K^96$E2YaapPEeE-Lg7N{rlbC+J-JUY70=T z001Mcrab4%p(L9cT&%*O9Cv;c7b>z#75w=@AVS|yQM8SM!~FKYJQd}M%@V}BdVkRcyjE~^!n~mis~{@Sip@cHFkEGfkBN#cG79& z;I)C2gsc_ijESzBZ094;ifAu#U$|q!W-7WF``+r;kRt8)D(^L}Ylc6RWh=~sOMHIA z`g9eWjX6+uv@|KtV}q1kOuY$n25tODD#v0iSb1UCDc9^Z8*`E%MA+c?u4g}@&x)gj zJ)q|IXhURuOM6P?-S}Pon|UGGg&2_Llpv%~$<{f$RKn(d&?^|l#*HEZ2+zN9-a*Eu zZ#;d4a}5o(E`p?@B$g9)yELni6P)niK%pQk7aYp5aAk8X?(id}r|UtV{M&kSrpFc9 z!Lo}p)@Otn^|La|6#LhXi!S?%NAW#6T~^jKpIt-3@LVJ+>t1ay9pJb4K@`vdXA&&l zF!YfE_cEC}gew zqRyXS>ujp9Z~oaxGJJ;{;JH8hx4P7MXTZ=n@a<$X>eEeFb0a4<`((sM?f~h?NFHry zZBkK)>zVoKOoG*cVYI}-NC=uq4!9qux&DYI{PRcmucD1&U`IDK@@pM`XfZx?()+*i zU%f#MoqQla#Se@>cK>WY%M*$DdmNRoug`Ko`RxXJAF7Y&z$ifWIRS9WM~Htu}YQg-Ct zc>LuqBfl*o6U8f75reFz%|!8TX-jDZE}(eBb2Q)} z9UHH6G4en@?IZV;#V3G8dos{rt*-^$&~%BFD2Y#muTHj%x}UmN;-a`E8rq~4xezfT zQo7A$2j+wBqLsAT_P?FzJDewXiXskWmuF*=_|(D*)hSt3<=tuzlHZFfREslL8teYh zvW|<(K(}})Bi`}}(-$Vkn!SN@IWIKAK|nM$idchG+PU$`r#l- z))Kowl9v0YcWJE0=EmoWN4!$bnYjk?ijXjd@}iSk86FJ&lf!zT-ti~*`yME&f5#*o z>3ej09kKgcjd44DJk0U#l=Z9Hx_j|s?oo9l^C(}8ok|<&ZMFluM}8S}`KVb6`XW!% z#HQ+Hgf=CIP&30|G;lim&cZ-0Zo7lM?(o5$h~L@U0lC`DSx?W{?+U%xeY{@;_~%c8 ziJKoB(MgjgCv|h34nefSX>TD{bFo2*8CN_>`$THcnb8hPO*!R*E~Lk}z}-)tpllBd z#XhYb<%^q!b!Q&uJB~%}?H%pIz9K;lX##K1&d8?*xR%`|mnQ$r2_6M$CNpOEWDlr! z(5bO2*g!Wnos0tNF9|UDSY7@ldz~#&25I(>>Two=Bhn?m!1MaVh~4*WIwo?gKpgc% z?P}oiJVBe#5!B=g?d4S->Z{5*m-|X zxZ%jXeZr%%nULTBl66gu{lMGGBTcv*Q`|;6x#+TzbYgqP-X|A{^!Z`gH^W0=p)ifv z@lw#%lSI%-u+eh{cAHm<%B;p8TkYPLYI|GdimVF^j4ZYYMcA&g8Di3W6#uCBRc)W@ zEO2Q36+97KjOfnyPfkg8X-hQ8%0(M*+~xZ<7l(=IjGi2U7hMUUV8G4_JqUc65(;hy zIRJO9cx|;;CJEt)m@RCZx|`X-RD|7}^7@J9$l5w!Z4#B__QGkikCA8Gvu;!DR_s!q zM#LvM2ugJxe{C{xFFqL)5KpXv@lDyhwMGi4IlXYr`=$QZ5rz+z+ELUu`MJLKqjf1< zv3E#1M%uy&<@!jsJguQLM~;rJ1QSUz_K8kFKk)T$;R%ibom`ku)#o$O&KV z0A<(7h-u;W!F7mAzYEwpV1DglJ!h?d{f?lp0X6-P>Wk7&|1?h!`6!$;$>)If;1f{g z4a)aBDN?k0SAv;F`zBB(AgjtU|8lxS;jPgDDF81JwtIdX@0Va&Xm)*W<*M}1^jkk3@Ja(2Zwu;FVPtp#tq57Mk7i8G%rJ9s?&r)LCts7xfIdV*{fiBF+VBXy3l zHLn>H=B-Frt^(fA)7^|C`&fUgM4p!wq^f+-v5tUeRm5;4x6D9Y)=uZ^YnD&Y1_NHE z%lbN*M--3Deu+U{un7`1$5XS!HZC)V$oi+|GxaknV1H=&wYzS`skvLX0>Ch{9>p>3 ziN6!}pk4H)`lt1!D@93bU18kH%_qj`hIO|G1{&(Jw}xq$Cu%I`ZED{x zwd~|clH;~hyZoBJz)zWUlyL3z*92aco7Tbvhj5M?kFI8y=K~KQV%9D*=zz4ks{`2! z+y1+$`B+ST=Gm+jtA|mA{vP`0Scy^MF*`xzQiv?&f!V|ISBQY|f&h^67!xc(nt`g4 zKFDup9?G32JffETN%dX(7|v=Y-Dq%F7dkfSHPPJR&kd!tg`aBE0GsH=K&@@#-QQ5&ps`czV2^&m3)wiQcSf z{#LnvZ9~w>L6V;4a0PmDi=r@Y17)+Y))lqfh8JW1&=Oyhj`x%t)aWYGU8VUQ(bd9$ z2-;PU(Rr>za<8?|7W;c&}FdSSSN${Tpq8n zo~;(V_V$ScK*#oq@}CuOrF*|D5^6i^p+L_pqji_|dl=*YOo!_=BWw!&h^Q_PTFhx=RDA%A+m@) z*OR64tpBp`c1ViAGX{go@EMomy7v?OIL*={W2TP$ZJ(Pup)&Y++NtcRW8@RG*uLu<3)_N z8yEwd{{fip%J{U&^&ZXViUVdTcr!ls><^Y~#>8u#%z8mCvlS@ASE>pM<8cO42tl3m8u^ovYbU58iILX|e^W;p8$#_3p5 zh8c`QMV;*0zodUhkd%6rZG&-ZjN-|yACuei^8DaPROD8$h|mey4;SKW#!wnVf#PC- zG_bzZ>=6TcxbxYPAhhnU;uKfZWKhgd6K;SW_lH(Og(6ycddWw+NSYDuATLbe_e{Mz zUSEnfF1}K_5NQ9`dI|j~=L0f2GSm0x_ZL(l(sb1i=u{RDCY0vc7Pl9?X7p^8qd)^Z zu%f9Ll#+)>(*t*=s&4+KXgT}H{v7fOC!fz~BYL%G&JLniTR2<(%=6Nb=jy~OU{gOz z0}+ZJ*dc7@2X+0@{CIyRGU@WE5V<)h8GZF)w6U`ZCv+OzMAoUZR0bdtgeCfB$~&V$ zxIfjksfjv4 ze6;%0T6QJqCF(6yk^vWPY)8@m)?+GjP5xIko!@Wjzs5(A=P3NpK%q^JP0gm}touJ-{Ozy)_pknOL;m|t{P&#re`+aQ^5XX8rwQOm^P#*o zPToJ7cIp2EZ2|BWdeaaSAiz7KV9R_&JJanPE>=%r_ir!!fV?!zKQhrHu1LlZc9Gz7 z?F-edn@gGB`S~Di=c)|`Ym9r_N*6UpM$yel{x7Y>w%YCfaR~ng=T8=5U`# zy`DpMJ7^OnyWbs;C@45ycQRmW3~KI1rbjIvYN-cg39$Fw>d{OtxNqPPVti~s_q}3h zyz(sWG)tC?t)1=MXk5;~wm%5i%0VHMLqiM8`E#p)*WLp3DnaS~B~vm2b!${zVxUAk zF@8P}I{F`fW%YA`Yb4>{eLzgW?slu|N*B8g))~kL+&fZUHvOO9Ww66GZ9+4WuBN=q zT2ZjLs&}o2zE4{k7;J;jh1Qpxt) zddRpcQ19tK2Wa+>(T=qqih))Cd<8*PI{Kx6SW<7INb5x78Wg4_~M|BRU!8Wjdd;8zhM!S4L3>3Y+V5v!u zTn+y>u;K^(_Wx^t0!ALXm}B9cP8%&}79OsX4{xl1jMMhosgQzs-WtitnNv@yg}+Nc zaZ2MYbCJjcPF;>q)wn3VKYWFyuNqI82tp5$krdy{0$2$7i6aibu zavB^?(g8vuxsMnv(`lwjH-8a32Ld}bc5cO~sG{g^7{aa4wVP-F+*CFOfGFDU!I6Sz zie<85!Cet+@PHiC=0G?Y(G+=o_$j007S-38T6FHI{A7F%3aC7&-al<7Sj~T=ytX{H zXlh17o&5fpY)@rU?L{k3lVLWw!ZG&?&C0Q{OGCiE%EQYEvB+DO6tlAwAUO8A@``b; zaUAwyB zeBy7*L1qjFbL`~?;l$VFnW#EePa7*vD0W;zq!RQP-V|)~bG)%MEVr_%LUSl9u&uzq zy*0-^qau7Z#qrJ9$&*&NA>pynzqsU;HQD<~>|zoR#aekcOjsm?9Db<$Pd}iH^5BYR z!XE$FKmY5#_rHDh*(dYre1)Yw-#pHwM~ez?6O|W)zAeN-$9wK=zho2}y3q5OF#lnN zj%+7PQ6OC!_v2W@xRRGS{HU95z7C(wqDZm3XhRMCn+3623DE!JisgSSyATDj+mWd! zU9GMYwD8Wk2s{h3l*i~9@sbqHY1ew(5!ueiMy~mt^6%ge;>-IXLlna;B5s(~T}P&) zWmk9BT?>Hxf?2Ewq(vl3v-Y9;PQBO>xct+Tflu zs?9ISRm_)NnI z`eXvMH#h3T+j7=RwEAWSV~bz*`An4m0LKiK5<%kG7VIIb^$(^~du#9~afeg;9JlX0 zqva%B*cWLj$0QZagk#WLR$@PYlaQA8)B9ak^Q@mBEg#fzs$DX#Zu;;s^sX*Z))}Z5 zw1{x#(rhL_O5YVlY6}4k?DCskKG(c9H3ZZ12-8y0V1qY`xP}<6hUz!fSXtk5<6S5` zbbUSw9|%UQTl!V~eso6A$20|=>73i^Z>TtxRS=+gm9>%~NBy1~$LBEvS&5jz0K`v_ zNc2wJw;a4VG0E$DnzZ_Xey_Xi^3pYG-Vg@XnWSVc#S@lBhTdd}l+ATJX`?}AvtHcl z=i5G=k8tro*>Oe2DKvc=TzM(x$@#dxBRvy$R6sD|hnwd4;@GSI(oHnzrihgh7i6J$ zN>ZG&wzz*5;&?X6)L^CN0_hh$m)Do-+@}nugmhJ*HiafR#1^| zq_iyOv$GFZ&A#QV#KzK4O-85lp~b|S9pz=Mlr&QpqDYWY9mtCa_HAcPpw?g50Ju}q z*!Mi2c`kq1sk21S>>9^`Y~EdTUYbd&gdb7$M`Mg%QAttiH6X8u%tP_oukI`#4X7%R zK|0U8%&3Xna4{dRUmw&}Y`3S0{A~Wv5@^JB=`M%gnd&^Fw@pyLIPA!Q*2I6(S7-r5 z!g-0RA=3NM-}cm|O~(i`T=uY}u)>*m>aq>Qlj*&$gWqXoO~=^V$O(0b8UHah(-Q0X zZ9`MX1kSyR>mPJsoHu-AOkCu^lUorC1PIV^^UQ$NuY`vhgY@ofMXc(4_0hI3x$TL8 zc7KNh5<3t9#H@d`yV3g6*y&wqC;aF>XM_hBrjNZDKBc5RwXt6i6t${gbt?fW*{8xP z((B%xi6h;;CEdK-mIJny`))Tn)_7>{QNXBp%){79+UbP!Xh*wW2*GWPP|?WFb0wMU zXVLmtI<%E-PuR`p@vRH4b&f?Lnypd0Xsd9^iOCD))r?y&@jOE32#AsEwCieigShq# zkf|n@fLOaxjG5{Xh>-XA$VF8uW7*55>3(PUO_5Hdyi40MpWCc|!htE~5jZH`Cg9Zz z>RG*zMo150Cw8gpxkO}g(R4)K>61F zTUnbC+Sq2KoA`&;=Sh#txr@)1AV}dR?Zv2=m$iX^uOD<;iS9=Ebe4pKIO^|37=hgJ zvU6`Jk7l;=YGLXA8+G;+kqx2#0+{Si zC3q1+9}1~0`1RaP&=D1J?oo~Q}UP+v$$pkL}t`wYCTck ztTsZ7Q$3fjwTIi-=DWGR-}1VwwfOEs!n)pWCN--{5nt*T#9JhAH=T--nI$^aC2v=A z!niC)j^@~abfhPXiZtxap2@J%>{WYy-wbtgZD(X6;{XRio_~|-K2&FP>Nj1FvbH4i z8TaH^wW%C7j&rRJir^q*@@SK>Ef8XRKgt4p59l)mQDg%@&)|@q-Kzp398oPZ@1qI^ zegP3$Vb`}N-g=KjB?jTlJ>Ia!Y7mrRbf4)pw zr1`)$95-3qer*ruVXDvI8qdrLSQ<~S1qCL_e#QF0Uez&fRr9dEOR!C!{ZN-m<80oR zjbSpMiiwh2Rk0K$A{#r82DUBG%RjFcnxCVNzo8U>J~pp~J>QxH>J0?qpwh~GT15F9 zY&!J84aP3cMowUD|DS``R4g%o9gr>%z6fs_ja>7r^{+^9Kl9*_n+;g&42@)PSPL~8 z(9$Ne>nt{8G*j8Qi&cx8K6ke`vVb>rD8M4YCMHgv2Guyym)SGT;|@BPJNrC|9yHLB zEJC6e-0D}6?R|q_FlZ{C`c3!yvS+cZeY{0e-3i8dbjvn}+w1vuFKh5b9m*0ER1*@~ zAUkkwGN&|DjN0=DX-tCY&+%b)tY#uC6K!L7Zn^zj?Zk(60G1+<3d92ibWM!dtlK}T z5`Po)Nr4Ai=fo#rP-gxM??*C&3>zlxpO}9?Q5V~BS?-hWQ6w0X;Rd~qBQ_tOe#!Zs z@O#aU@vrJ#24`pwDDScMM8biHAC{u&?k2T-yI)u81_psrBrio6*-(RXlfEpNxx|2A zC=namZ+i64er~H57KU!8N3-cycV2V7`whSm;^tHSX8r^B#vP!U8A)Q0qukXMby&=X!S&qK7lP z>G(vaFU`KjVxl|~>5wnhtIG6h9US3D38=4u`>1D}(ddYF{q6IwD92N0Pk2htGtGer zpt0AwNUn4en>p#wKsE|cH|o><6nUa7o&KA9sFVmUba(}6@eQAEefxKS^HD*>S9kik zMuRk0mWZ&E&*d)TzZ{PtrRVW?V=$gZXFl{2f?(9ObGSem?~>G|a^_bmGTxl}YDFBy zcvtbmCKOWGa*Exe64D_(-u;ym_Az z!_tWyHA+X_w4H79ot>JudgE@kofkR+z<+;e>05brmY6yl%?P!FmSsqNZd^(59nPk1 zlJngy6sRNyqz~FX5ElRGoN1ceYriT%ib%0X8F&}?Qai?}iR{MvGhvF@oaA{}{K94V$<{~{ZJG1nTYS}_NGtbjs9TJ@lY8?jyRu$gDI zOoiPSBv$%l!&je~zGwf($8%KSBl|<^dqwWF4YJ2TA6j-aYu;h5xp6Dy6bEuVgipnW zj;Q6=ac2~=N^GC#t-y}6-|q9*UC~D>9@PZevVrv#UaJyns9g<%=QIEjeY zO!3!RKW6oC0{_)JlE^9}(Y>8ENiJe70~wEYQb)WpSY2wNT2mgq#Rtvf1yYeLA{jwO-^-;O<`__p+yHjT1V;0dg(wUKGG%0G}RrCffximmp-DBoLX~x@YYBo@9DE4 z&W6}>`=S+Ya3kKL5Dowhl!kZz3!jc+^on^1yZJmJ<_zrT8SzIkM8BhsC0rM?9OSRF zIc~YgOSH@6P38tj@2&jp>dyRd#2#lTFW%2`HM@EB3fxO?edg@Lhp>L!Y66`;6=obI1c{8d4qmb}*bIG2_<&ac7jW z2wB-HU41_Qs8|oEPvk{JMy>%SWBlUUAm^InH0^`WmvXJkL5U5IjB5}~Hs4ilaZgRd z_>vG2MX#MoSe+rE^CtHC$=dOQ6klH-7<&=;bt=rqLf`l$)ZF00AIm5`(rEUH#1I2$I0bfuWx2!z5 zgXcY&Zuo|d=q>0AA9IF##-X$=ZPL*sggZ_*Q_GB5J_Ff?tR#N?@`x~iqKe3ao*SzHQ z^)1wyQM==de@Rtss7uStI#5}So(ku&<@8uRAeTMifm8|_HWIqVsrNlq8&<+Pz9K5= zM0aMUnxzI9416nEzS7l$iyZ*&e`VAb<%AVF<*cKOB^ z`otGvyK+FyWdN0lpf9QbneU8_8!-;|QF>3$c!)8rs!SOnv8azZ>6kh&ZB5e|4GIgg z=o}{v&#U0xcCSA2&`oUgeVl78sj`2xDn1xdPKRy1%p_-iAtj#^&Pj>A|7u0Ec9N8Y z@j)hI)PZMEUVDc4b|VBQ#=+#T`eIBX9(3$s*_T*IPcV3zzHi+csa*K7KW#p^0(+{@ z>bi1kET)~ZQ*2TuX2NfwzwBG6d&hMQm$y+;52;SHkG@Qc$&1uZb1U-foRbX@>W%%M zm(`V{qhoZ%KBLH+OYpB6oe5IC0V_m+a!WP$aEitc0RC^(pUv2G43L}#JgM(SDA)(dcce5y z_Pb2sCdPMSH|}pWddbR4s7c$tQ0lD#{tLXH?C$kRJqLOJt1MDQ=P~i2QOs%@7qZP$ zybh3}a^dknLAvl19(9q%;D!iDi7BsTgo7dDT_WR!W-9C$ZfF6RS%Qt zJy9p! zap_aY#>E+k5^4njZ&l~S4<;sOIFZPd)R!xV_T;^*4Qa1j4yiVr@bv}CxW3e*Zy#jD zw8dWr>U6UwUYY5U^(^|05WKOh;Q?rq=?(Nbpnhr=bo#h*RwyS9Z7Fb01X<(n`#I<7 zH9_OxJmCLsW3UF#NuEmqH$7}VfRs!W3%e7oJdj+@?R=F=L4M(*yO;~|ftr4NF0^{9 z8-6g8f*8KEtWOSS^qaa|Q2reh@i}jtanEjJ>vSBi({0uBj=mV3=@BsS6#x?kNx?9r zK{kI0k6r&(y&wgxJv1XkMVuArWI_`2?ENx2N#rcwG`AD`vpg%yq~(~VUQkVG2=e7l zVy7}^Zi=E~)qvFz0vVOtJRMEWD-KHj)`b$O73RVs&ARTWh|v%#;i;{$su~`%8)*;C zQtx}?0<)d8I36v#s{`habHcKN5OUFMF6I7%aC5+s6-oDoTYft+gx{Wbf)CFn zQ6N$Sa<7%Wfay@<_;!J3!+_;}PgmcT|I(odXn@VpUEC0N!l!5Q?GHf^M@BNTTZg zY=-huESvZ6rSEf4Z#IVv> zn=smdJwLCV(?}&c>|PG++>J-a$$Q|HoE8EzP$M$Npolm)~#XtQnr7ddu!-c%8RP!7v~8T%VSBQh@ak`b;!EoMt$N;ZPP% zJxtb_UcRAzc`zO}d`PP@5PG_b!Icny|kDBmppkY!f3{H2~~%rJJB!WTM$aBvBUO zB|P^{?RejqKkvt1FyYPy5moOM8am-u-mVy66GM}o^c<@bo%_m1&Ktbrjc)8%A9$dw zj2aeXnd#lxxv5aI1Q?TDXfI8kS&AaWQNEqFTtPWsU@VYnUk=lJdM~CrRq6Md=tZU* z399J23^S~<^g_SY$(W5JPDfi{uijls)zd(^AJ+!v8iyi|mz9&V`PHrsJwLpO;f{8% z5+}FY>}E7NUMyrYr@xu9{NYiNV!TTl(qJt8*p0`m7+QG`Ng!SH<3_Z^M%W4&@n*CW z&F2~^kNb&AmoVHU{)9i<8xZkaT!D(mtNycfns%dycOU_qP=zN&l)iQK((dJl65ioh z)~5C%a=2UW@iCWh&*`>7w+IaJJo-SN)Kh3=*Hb)TWte5IIKs9W*KclO;Ql9F2lgFq1N7>tz1TQ}@p zZO-eQlhS0&-^E6*0>~|+P0nlKsM_vxg0_6Df2XzyD3NaOsWCA@xVR3L1NEt8G%prn z=kjFP$VrSCp{{3&m0@*Ni~$YIdT{<5^6-a=NNV9E&FjZmg>^8`Dm}yymAy=He<$A& zqu=<->g*fc`ksp3mXiY6f}iBXalSe;ph6|fxTmawSKbMd)<%xHJLsR}CD%#eNLZhq?F zpOJe_?B{^*w|k%1h&#LKt&L%CdEfW+D=`)qy-hbzcHg8I_l@b3FA^KDOX=;jf<|(9 z8KWok^i`mAQ>1JpUgA1Aw%crd^)B=oUpmex_f3!1amgAs%a{PwG!e5fzIr>}Ky+okjm*tG%+EDmr%K@|kC8Rpprde4h_n`i3Nh<-nFF?(({VO%weK%9{d3 z?l1F7JJwB~aUb;ZG&MHoQQVh9s$}CZxNIHXmg~G371yk3gmn{$NRI;ME(FY-r)ir< zp-D!+vU2h|WkdP!PEwbIV-{-xx(*~Y5;eC9jc$cHe`8lMJfjIo2og_J_unIt9@B#h z&x=S*RWqM|kpbC^z+-MxzUm(rM8wxL**MCgf*{`nES(whd{*_tC6AwIT7XIgYw}I{ z#j3jvty%yR_q|DRtLeF38X&T~*?H|n@vby!fsy<#uLI}1*GNt1h#0#!d#JD6LU~5+ z!Zj`WX@qR#_B$Lu1VDk5z`#<7X#tu1)u@|#Mz<)No!1V{SPbu{yU#l_?6VLTQ>0Cl z&^xTcx$wIyVbWe+a?IBkD_`~@cnOVKl)pxGpH4mnYW_+Cvwx|e=TEcU z045MdZKS>(IOr=40yq;_YT;Y|49ufyn#| zNq{N#)~rLSFQCKxNCO=USNRx!P$=D>N~RJw{c9ob1ugIxI=qz<>Fzi!1Ng zm$1A}KmZA3^wgZbYU3l$d?3c25o#8e1EW#qf@LZ(z}G4n7ww3*6c|aX8xxS?B_KU9 z?kiu6)kRK3I!y4~%%vOL|#=`;)j2~%5 zR}|l|0=gjLInWs(OAiOmV$qtYn?~EjB_kfn8w1__ZY^|r_E{&)bkyyHpoEva-6F39Eg?aj?Yzr&?BWGEdl{! z&2^9>Xru`oP%&9Ab;NK98iOhO;Y2v5 z;PZql5<}Wl(aKax-Vr+Amn2-Qr5FTv8AOK!;xnL_TRSIHzl%B zrkm{=k@yFjN^YGD+75pUiL}#RZV4b3W`%bz7ldiL)3U4RzotEc(&I6cshX zZCvo8%!iTQd{27|+6JI_fg!?Acr53ii6q0EsOH_|g24QwU)H7iuoWN5c@7`|#XpEq z7J5?R-q3WcZGxPjt(GyC5lNTWMEt<;=u0;ZSqU~2lM;GW3KechG%Ub7a69jw>97~3 z;vd>zzaZq8HL|iaZqf6C5Zuh5sj~g`KP{tYp4ez*y+m#y#0NvGS?#B1K&3|#dX@+A zIUiG-0hC!^`pPD=8*tOm4N|9(L7UxyS(L4f0;zjeVbQetN+{(7Epq$1;Y&Qg6xNtU zAhRQe31xeMF1Q2|XNEZYz0g|fj=AMEe=5%jB0TJIeynaTSS6?NHVAjQ4#7#MkPlu~J(P`y*5s0*lzOasgMuJ_Rl^JF`EkYP~A^vps^9=bW z)^$<*-tx=f&>-#V1WGO8tKUZHO)78p3>a3<_?{|Yv{}1)(zOT54P2e-2e^%?BZn#q zQ+@nv`9QxlmO~~*1g_ZpvPNYxm(YG!%})HZIOY)f=Cnc3xk?Fw*iM(2C=bWvk%HNh z9B9HLJD~)GVnJo&S?OJ#2U zfnjF7Ns*w0N^Ds{H|b^%4k8l(|<&BSm@B{RZc`@Lv0l@{rD;v64=JXW(qVsV-ghWB#V z-M7$WY#LB92Q(xbj=8wFv`_fX{vcqqvbDSf{mgM@H#aXRD4~`d=XE+@Px%Y9Pb`fj z&YpnzAo^FaPM=pp8^1DT=|$t%NM~YoeM(8fNOax6AuWvvQV^bF*Zn5WJD%F^ zF+sm#_v_inO+N&m?F8~5<|e^U5CFiKJ@q9`B~9&i=v1;T3!uPz|Mshl^IqYZ&CI`en83>L?-itReCkXd=B??fXMbeT1toxm z`maT>^yA>7|7w#{MxBB$fHrF4&YfBVUm&ai*c^LWo(Yh5s&JZUrvSi)6F{pKE?$KE zH+(V%WRTZOB+i%~MyXr_589keD0^Fa)ds*939+(L>f4z<;0H4RGSY25|K3MIC;a%}(-(G4m%d>v3VDD8R4zA(I4Jo(}$NJJw z-^PmlILv5yY7&1-NKkmtHdQAd@ws{YbqHjrI%!OLztXL_(OCC3(1-4=N)LvYxI9{&_^{Y3oz zFj#<<%B4zLy{c0WNoWM>_m(no?8O~6ESIXf>c$&}2tzIZ;+rl*Ra#2r)eyb%so&we+QPAm9GEF!K}3}@{AGNXTv*-bqj%8A?7hivEK+} zQ;aj+%CRt@iWX7$Ry~%VcC~aoeTBI)a*PBQ<*I@up*FK~szAZY8COxMB*QTS(R&46 z_jT?^>#o!-I^x>$p8EoXF`OW&WH3SjI%M_+tSDm6`8okV2}rOIOt%@ls32m>tf1cU z8*PwPzm55(<<#<)!{aMF-cX--^l>5UokrLm9(H|Z9w{f3l}^0s2w2}4VPdO=z;+=U zt3VQ=iz859u>>c*U!+kzuCCZ4L4+@-HTU-MCbR4*uPSntoEz`32gf_=GjqJ*WWfA^$6HD{ z8Xoq^&QL+cqS7T!+mgjz&TcO|4Lue>0Q(%WhLKNkZ>Pa^cZwCcYi}*g?s%KEQhT?e zwzw3PItt5nBUM*=xD^IyBM%akSH4f){j97uE+xPrfaKwIMROXg*KmEbeSK-sYT|C|^EU#t%tN2JD{f}A0pAIQP7HMEJ z0Yn^n5T3NF1JIc~dHr)>Z^tC(J_46g`TP9l50<9?Prgn@tSkPFcY63Ih+Sm+4dLWd zuf23r9`q!=ICt_CA`7`EzW?fw`M?!@+BYdxWo>9=9x=-LpL!N5u=fh`BTMi_8g)UP z_^}Sky$whIU`>0$3`pIBii&HPw?Lhy>^tz}kAKj#R`McpIkixpD%5B82D2dLUM-1A zr}V7J%JT_z^RBB)qs$~8d=KhSSlgsc&DJt>C0w7ea{ZwJS?zSC((?zL)Of#?6PV%=vE&P^2K=c?juKEy`_y4! zHZFgEKLIcOnoBzrOs(em&PgZCldtlrJQIiQS2)pEFoyew!gPOIIXi043n`ac${xx$ z#hCOguyxP9it*ci?Ly$0!!>ERlRa`?CV7d=+ol&3Zm#9jPI;?FZBsQN$f6PabEA~9E9;J;jqm*MA2rg$=V^+F~h*s4cHUm z3zVe?DrF6zVfUb_~S?F z+#0A%yuinfO8ha;OQaXxD^&VQ-Ya}hR0=eCg88vPyw?yXaYeCO(W*jNa`9XD;=Ae7 z>KKu?LoKQ`PY1SvHg6~nA<4=HWK>MB%u(dfFV%g3@;lyV#=8zn0Jb7#S~@J0N~Sl4 zT_u~Cht9HGpF+M$9~E6E$};7aO!TB(e}$Qn@&z9W+4W3A>M^7!7uY%hx%R~#X_ODe}h#HJR6`mdMvkB#NVfB9)y}e4rR$_OmSq%3o@EN&6HI^FU=~U z+o@kS?h`_}!NhWZFJq9d4T_v)3Udf~T#-Uv_iL{I!2B@!$#$g9$2jcLT>MfkS&*H>#kf=)bcoA{~C_NnV=6YDGwMHRR%?_1U7>?%@ou_LPv7rVU z-VrOdm}owulS;HmT0}BAqMNec(W-$BneDf^Mq5zP&O)8S0Dn0=$E@p`!_H;uZSTY@ z(IiBGdc87BOXM!-c-JWe72Kqo{Kc(}7UeU^Mj~q-%XtUBH9{4vm;B}IE*LgKcKyM@ z)r*X?bp3^(Lx!^z_o|3@&~rhzRljA(CIU@qXS&^LQm32w8Z!ZFJI!j6wGlwF!Cjgs z0}8!@f~r9T?fVRr?v91wpJQ)dkQdKVvLIV+ht|pTdmqCT!|u z$(TSkM)cc&%|7>_T=TqQd#=-&!y3MQQd*y$X=j5IuzBX&vk4Eo-KDc2cX<-?j4rXm zylzvL-)SWDtR5)w_%#tMXIg)B1o@x-x3Sdj`kt(~M?+6oJ~sy(w2;SK0+%gXFk7q+ zKVV4@P>~Q+l4$<)>s%=y$&UAHz@2mQmadR7jAaZ!bB_@4iz!fv^NFVFO_7*@crND- zZ-^Y!%%f<=oUQ4jXbx&?e&a9Q+4EZw;^>t7B;xlhs!H9=LfPAqWTfG z#uH|Y%`U-2F_*f=v-vNgjYA+4X&a=yze+#vDBmp;H)T2Bo#3U$Scvs#i|V+o;pAk0 zp)q0f9rHq!!+O0PjaLb@-MC<+Q3RV{d;bOySZ?R(wq|Qu*EXdaZ3)a*Og_olQxHdV zt{P?zMCLRF_y?!MUJXLjK%<^gZ5D$+@c4q4L+p4t z1Ky&^X`lhhKDEtjeiR-T1xX>kUCiMxB6M9O7O#hA?D{WPr$@)K8tctTUF00^-i3dETJYVu=(B+!;XxyFt;c zPd$yij+=7b6qxN!WiGIfI!0y%VHa6$n0PAL?#Ylidc;?lcyl4v*Fj$=ov&Z?H`-C%Q3xuHp! z=PJ|!8MwA;E_O3(1|1q@lcNkg%!D;bTJtb}mk9gyA?@Ud*a-sBzvS*u;ug_sS={E) z?1V-GXfq3{gfePl>J%{)Q5Pla&wSrG9~^{oyOXSawMt4IE+;S{qJ!dsQAm=elDJ2} zYI8S=rx=_M5O_=ryDm$V`a9LKy0_kC>!vQz=^?YRKBF!mCg#2^*IdRCS`3$nEksHTsfy$)6zR=4rT9VTD0Nd8 zBZ8Snvv_@n7#UW~_#7YwuEA*bz4R*zt}$e(PI9+TZ^R2CF?4P3cm<-`5n z`1OosWF$UoG&Jv5zKd^4Wd%Ro(8mZZ$UuE~sHp@4QSwf4JdGTu7yS~4? z&A*&Y>cSD-NVb|&=6v|UyzwC#5x3x{b0;^ZR|T5_@@|~h(J*l(4yD`1jK^65`A8+93bDP{m9bvj=FQ=;`Sh6C*9aH!eb$dZR{I8wl``gtS7}1*s3*cx0>1 zysR14gvbAW-QI|A?jfIZ+jf~<@5=9(w-CsG;TMK&pHqZrpaFncxr8-CVqtI%%GaoB zCqW0+u554`7S)OnIlj$|Y3lmmt))+Q4FbtN7!2k-AW=uP?KUW5`^gBpI53gW2n3(K(^tI|g2Qu1RPhPmYBo zoB7X{yE^0ssDGPTp>YNHhw_jdBQ{&=vKYVRoW%z!z;AzdzWyU~`u~N`{Ac45C^K*m z!V@aDJU3c$6{Blb`6fNId42dO^$g8X4wTwXLWR=`L^}ZYo&?Z3-|Z;=Z{%v z@V9G{7>!Yv`t~6feHa%7$OPRiUsCchxW(zrGsj%!b<$$$WxEd3(gK(?HBbl(^;Nd;wVcP4krWG@MEG7Fo0Gxg9eH{w@M%)by%wIJcjYQ*( zTFs+sA`}B~to8v?^!=3c%dez8wX?H&MiSyQee7D`tC;V^jQe*tU5qXjpHb}Q>Am&{ z)eAEQ)*fJslVk@!n1e|Ke!)1#-qqb!VZv*DPvyHS1+tQqGI2qx1&oI>ZPNUfc^m*X z%e|&|%5dayc-}-{(;G~ZI|OU$Z-mgvYIkkOZ7UDkb3S;a9xJM?TdzXtnIWAZMN6GQ?>h9XT95H-jW+Sa4ZyLA7gBLfM~v$ zaN+SljH7wpNOT?`w&eIE#2#Zf=hE?`FuhvyFz@A zCb^fyxEB2$|AyeYYWs&R+mnM#vtJBIGSQyHUOX}y&hPorn$cFE+Gx$ylR=%c=JePn zX3r3NnJ;?%hl3(@5tTJ*{2`qC%qU#BzJ6cDRJlP~QJr1{gn!O_0JuQ73>Bk^b3AH` zXYlZ7bAu^I*>FOrnlbv@Vytaj{skLPI?l^9{~C?lg=E)6XCphufz+PIZ9xZ2F#A-G zUL)`zR)nT22LM-3SXPypdCwR(Nr# z@@}WssyE0)P5qlQ&gqEFXA-B0mGtRG3FOa!KS*gU5&aVbQ~`XB}Pgo2=_MGC50 z-H|`VY^QN!FF9T7JK2Awxx**up$9*A>h8rlQXU_tgrC3}(wF@!{RZ!iqe@+&Z|aN? zF+1+HJaTiNTDcQ$E@ZpAVIGoxIYutT$}+E%F^-)1yJ9vUqq;bkKSI6=WBPwnGbgYJ zT>4)S{#9G%zuFb4%yyyK2W(8*MV5KTyae_;VMfbe0(4(j?h^dd|8zv-KfHf@#KtTw zlI)W78HeYRyZ4K|I-8r*l^Z&9v&sUfb4o-2?m0-y*dHPADqW~f?rcIMCP;;PtH#mV z>syadoe5~JIY@C)0A=k-Q0ar8OiT?{I|?0{-W+F6a8GV;W%~J5#Xi((vdJ=9xL*hCt_Vq5qoVfoC;TG?=+jfaXX?VQ)iTPr+guZ#k3VUK%EGXUCisP!kbSXpx)e z;4}c+&74(H5B81as6Y3P$B&Xy3e()&DzZ?66wzxBByE1*rS8XX33t#tq*3C;5ro!L za0c6CmIOcn7chusql$iw-YKwuin%FQ ze%v3LF@iYTTpdJW&fY<@!<5rWXKc3z)=q=v;|0egrVJ;Aho^$J_2lXS?R`Rb{04};E@WXd7oRwM9BaeF)omFOBMe_aemnl8jP9B3=@=g#<+(hx` z!wD~_`}xbRva+JJRY2^uAOW~UJC{yb{+!Nv`psQlg@F$719h^?C1*<&e>q~q;vnNT z%2r}8SLMW0iBKZ62E*Ol1VGJNn{E+REC(42HxXtC(5Im-a(Xo4IMCiwZg+Bj9TQXE z7v^c--t*ch-hgj#MY_Z6O~#)rn(@l(Vr<3tOlOtF$COM?7PNJ?jCo;~Zn9$658R5~@MeaE?%;U9K;L)eUR(nC2BI`A0ECW?in?MmyfU%Zpk5aP>eObsQ#P zVG@#8y$f~ioE1*bJneBRo~k`%UlvF)70HgC*!8hj6ERedb@{~!Ls*!TW|CH zLYo*az#SJmswHXl8WMu-Z#5o?963y_ZJ=}}XTj3EnYh{muH(SrfM@2%idiRU4l3Ae z#2>Av-xDW}RS-)#mc;twJXn8+dKqqiO`t`v@1(V!t6UA+f?e6GsS47ibkI;Wd#Mp5 z5wdW~%HX|191bZrEM)e{eU|RR2gTjX^G4D&?-kM}fs{C4RD%dDJs|ucZRG1l=aB-n z3vbPh0W%icn*BUPGX=TRC5+^ff(DkGOXd^IGNQ#z=w-}msSR{9wMx7HRm>3+!@@}? zjo8#fC%bywfw^Z0zb>b?b~XnG9(DYvU{s#8Hl8X70}uTJ|4*@l+acSctcZCt4bp$f zr98m>GnTYSpxfRn*Z`W!Pt-Z9T8YR|(1_=D^fsOafUuHt5$YPFIpVB;!>{3Ufh$)O zYro}VGnXh!vs!bk2?QDK`RJl#E6VSvVDhFZ*vyNo+wwQl8~2qwFB}vnI*>^olH_gL z^XkK+p$6-d$rEt4w49vPz%6& zxIh0}O7F3rhAsNXCLjk4x;trsDCEmDWGq@t?oiE2{Zt zzuLnK_|rf*9Hj@W+Wzhs`qz=2C_H2X%k?6hzR7-j3<{PC$}xz*EHy)_CXy9EI^4g-&)+1!{78q+Q6TpZU}1 z`gAO%4*w%V@a^aB`#0F3_x~oh}Po36l-K9dPC@XEn(|?R>R^ET8yR>RlKF|FJg) z8scqbpi_8p#L0c`4XgaQcGk>xmN!c`n@GYay7q>i{%-pMTDi|#*gCAIxRXB7H`+Vc zRbZ4}Wtu#huJ7`3Pql7h{iZ@90_thi4h z_u&aeNzHR3)|3mqN2z)TnABN>Ckxm6@Sd_6MZB4t+^Lfac|myJ9YmLG3ql z0SJYuHKVuoHE!dwp+|do@g+IHwA|xKao(j=RvsSl4>HL=W=uQztUnNSYVF29oa4)L zLUBQdJs$1GdPh2ON;}=@&ZhtpQ1qb!?>{)qi#tTA0@3XeDof+3CH&SEOifLs)UrJq zZZr~tl?5)LFQ*peIS*9dyT?4HuQ%%z7FNt447v{lOL&m`@l}p z)it!AVNbvMUSXaOy=Fz=Nk`>z`vD0VF!^)lA<~1I@xbqmf&~BoYgWD+E52l{D36bY zfa3^>BejdpZXu+g*N7W}fmn)?FgRo6ST##@Ye7<+_CRULEgRJOa*+*e9JzS$0#qu& z4#i0@;0Sx~1yyW}nTRO{2W|WNm{){C@QWhJdxcwt@~zmTz;X_54?qas65yD_PtGvM zu?eA`wr+If?#I0gl1L*%ekr1|WqnYC!!~L|*|st^eoPId$XG9`j~MHxJ!{12T`R$s zfehi~5PA&4TJDg2=>o0?AQ+Qz>L*Qr2wJ1mWb0O-a%Ch{oV55}VaErGjv{&+>NiwG zTmIo5t3&w6eOC?4^oa3|wL>bDU}!vb|LnqDSm%X4{OrO^9r9hUx7&~PwP}}A&mkLL zbha2k|NS4Ifc(clzEYb6cL8}`pdj}QN}9*6tjB=|uPgU^0sr_eq1;;wMk^`{7^x-) zBUD-3@CY@a9<#KL%rs-ymfITTt%ELQo5s2VvX;&0(oQ+&CG@Q^LB76Vxj+xovhgi^ zK@Xg~#nu!68tnb^@S)e7{^$D(*7Mi#r2Nrvn#=xs!ztFo4)Ra6AF<`%ZR@_zZT%=L zeD2qOFy?$7!yx>Bp09sg>ORM&e_$Q@9ES@&uQ?xYML(}OpVyo}?dktB>#M_#u}#kU z`}Ui@;>~NFoj05%=N_NEm=$o};dDtdGDp22$0By9=ilAG-hG6AZlf${wv8x59c6l0 zPpdn6!uaRgG1PgjReU`Y(KT>_x_*rn7DR+d=RTk zEsk{+eg$ysH6zZV(3v;@128F`;??gJbW2A$618kl<{Ff-@o1*SiO@G06CT4yHVThQK;p=pA+Oh+!-f43S@=|md!Cbw z)&knC&dO8nTV}k=P;oHq$yE9A)?|Omg=z<#fn~SGIyV=Ol&+dpZ_l}cQ``phfngNX zJ&SXSZuVYb%h0qtoszAS@$dv*H*q8gTYWpYXpXVe`;3*P;z>r&cFwyF0AV68xX@_y zkT~EW%j1F=+UkzSv)h{PHp%mA@$0iUo4+1~8~Y$w)WNpUTsZcMc4cpYyB^o+sO;{= zUSi&@-3|H#k_PM;tQ(%0Hw)~MeuV+;drvL9CSp~;nTMWUXBBLCc!cYn?hDvdZdLRY zZ=s$8nPKHo>a?q`JHhX!1`9Wl`YkkeT^DeT;`7p6Cs^~R)_U8W?p<`W4t$~4saR)J zKCV$%2M%2+i4y&$z{QW)TYs~z+!Z3!YML$I_gI>lAt(B<=kI|9?bw%mP2s-+B0N z;^|+Mfb;E67GVXHV!Xi9ogC#yNbp6_C7B-gzDy+wCwZ&Dn0fRDDp3PzY$Wd8!2ybu z>>hM%p>gedg-i1o-s06u?-j;5;=<(ukCuYAHVh+)vtnygD2{lquzPI+=H0d+kPptP zQ)U+02^hEJYs48Rg;^B2>Z@QTP<*l~IE3((T+m?`d6G|}ueEAKbQ1Hf?;poe#Zd>} z7;T6Ss2zQ;unQj$*nr3jXgJ0>etQBcM%hgzgM^(7D(Hb;(jdEj-{m+i6kLHSA2>0 zV|EErOGi!%yIaxw0~%GXS6pzTMQ(Ux^<2H3!-R#9IyVd;DDai6VRH63EmrXjHXglR z?6;%eYoj15o{Oq@8T@5Rsja_`H7PQX+>nds!sF1Z1xG-FnX4&)C)OM+d*oy*IqaCg z;hl&o&pQ_k&!NENi-&Ir@Jn|ZJ$y=}a6$N^M?qCn?%7XMf;@waHj1-$1kTdz&X3l~ zUhqissw2*9oN+BDn?FEEl_G_k;lhaInpl(b{ZeQE(6CUPa#gny+3X|sAnwoW!+a?X zph%r)KN*ajkTJ{kLG7;Vpu5@=OaeuPM5*neKv3`#wUnCN7t?yys>5c08H&8SKJV;A zs#9{k#yA33Kz5CB5kxD7*ZO7iN%g0b2%;5Ja=GV0J-eFouOzx;935H>^kfEnzB!D2eQO)--Z*a%x3u87yz58$%z|NXj$ z%Go)r0pRemlJaf{f2D4XIU;E2J|)y1Hl_mxALt?mfQ~8+7lV@T8?kE0k3b9ctW;r_PY!7+c}7`iEj%Yn6Xg z$=yfr$bn66&qyqni>$flkik);$*j3XgajJphUxen!Z)}cf!QFnTv5tOLEa01i7c;h z0w0qcw+?6=q9BKllEd3MjTsYO=y3UB*f(wCM{mpMZAw`V|C^5cC!vkMMFsu;$@{AS zfks8n0InH#v!y$sbEmDa3n^qYO$5!8L!ocrfS;KhAvuh@FWx6h1D115-LENhS+yuZ z&XNPJ9hOszQM>zIp=xxTmD1=MyW{tb&icGvV#wNKBRbHMujj++!vay-p98=12k zntMmZq&i=m3V5@rA{M9ax| z@X6!lR8A|)z6%&ct^K-4szojEs-jz+9qH^=sl98#EJs|fArTYwQi@>#s%EmWq|$MN z=JyKgk``9>-cvpx9_O+hTisoMzSMVRtTCO@=I%7s3|;nHAF;02VZnu9BqqM`JTuu2 z*_c21Btp{+sS__8M&`0{Wcu+JuXuT(tnArf`667X+k7|5-zvPJA_F%) ztAo48wv8x>Gn#+?|g~%pg=J3OtNauHpB|*8aHx2tddk{4NMg4s3 z8@sDgUpn4$%f8ZQt6|)s6~p=!=vCnVLEV@TAbV@ZjtFiTsWqj+dHRvE@%sbeb4nlBf6jm{{t?MqL2q<6m89-j%By^H$Smaq<9J_U;^j@l z!(ETtHtv=hPF_;zR9UV2=nKkI!eQ}ki4AJ?2!-?|VFbTga`sdL8{wVD%l>9$XV=q^ z?Mas8?W0yQy$qETVLxUb_B36b^Xs{cdS||NGX8h9mo$vwW`cy}~Gc`|Iv;>@au~Q=$tN?-ihW0GZzcWPnUI%hi3ASGHTJ zyjQ>ga9$f=(_R+GKwKo#({K6P4FsIN*6a zH>>~TLQrlpMd`i5D-iuM9lPb$_U0zb-b>UuWuF$z{2W!j${EC~17Ml_sKrJf-z>;j zeAM1ge&cW4EQp;S_s^ehmh{Pm{-K*a_(|_T1bqiEuefgrk`?H(4j>_Z+G6R!h^N$* z&D6O^AZ&L$oM^}BqnG4I-qj)HXjecC+y@x7zPcZ^xbTyqHo*DhBH*zbioxhS8XRLbm*MXf z6iPuq|5or(tFa#qwZA*Z`mcLKv!K!7Z18_+ie37&-JkR!0!+b=r|Lg8#hgEC_a{^A z^QiqiYX4IU!{<@^Io5uTwg0xA`5bHixPkk;L-?=Vgny2;pJVOkSo=BFem;%;j|Dtl z2G5L|XN+!11%7^<_#dm+xPPB`>gKTV+9lG7E)E|L5VJ2TOvzN1cSpKEGqZwF3fa)U zyQ%7~Z_=4Hw-6OYaS7`;D4(*6D*gDdje8`H_yAXNj}dJMT+h`Kqf@zT{AB=FZ4SOw zRe8ZdYs16RZ@VGxFR!-Jf8AlX<-u%WY(aL;puJl9j|arRareN^;?}OAZPl(AnJr>T ztg+vNoM*4i&VM-Px%b}^YX8i?|Jg*{zxZxENcBm(c4yo5RJ}>$>*fHRN^#%BaUVsY zed`HwDi1el)CoWAF2;7aCo~D>Ab*+3b^Ynxtc{*H*|Dysvd`B)xG1F}?`54!9sH1+ zH=;+1BL~Hh!|CUhd!qEX8dMf^#HMrW-g5Qv zkS8YK7}YP;XSwKnxU`JyF(DalkwCh&8>k(i=JBWmy*Dt4jAg^C28DJ%t(Ye*cZ#6- z4q+Iq{BpcVdWxgH{tjHta>t|=qs#JjXf}LP45B=6Q^y;`b+3%(1{_Eb)>qegnh(#tbfOL;4M(72 z71fJiVCV=1YLY@=GAMIMJrQ!RqV@jJVQnx#&X%2MHyW# zbBxZ6adtfgo`sj{${ta4q_&h-S_7t8^caf~*`~QUAR@b~EI=p@cdn`#m=R4;uZL+E z@uh97dE+W+&9$USwyEng^tyaMpbF|?MbpZma?cN2^6uN|s|v z{W)K+U$;JeD#mOCx^}vc+8tR!i}meuuf}Z2nmgV}l1Ela$*|Njk=Xtfy%A&zmAI5F z-yZWlL#JmkR`G&>ABTTEWs$3Sm&-meRhSyED;3s2xAE|Zjt{UoFGq4=%egqzyeL~t zdr)wjqT5p<_%(l<0G_&A&^+Z@dp7pL%Awo=GvA!ct!NKVy;x4(gnYK^d^LZ+-UwDj zJ^Ez_jRAW_p#@9yHC&X{O-B; z*?;ilfvk6tcV(^he%JT=8H#t6rwQwJ$xct6N^>8vBAyWP*e_=i^PEm_1G3M2JxF2d zg#Cm%zhr%`Rax$M&{>i>iG@tHE^|WKcII3dq{FmsFKcL&cpm80`T98*$8>wqbD)9J zSE0;%@N0a1#f&DESc9>_ocDt|M)hy3E{T#e-8Yw^!sD(5>-JrMajHnqYf^_?h^rWL z6yUyUDdhtet*Snt@(mMseNtAOWWVW(;G68(Zd<>7bQV!_(fUVIIukL&lJ+h=8MdJC z)^Mp*8v0W}D%M-EOA0Dv6=KXgoi-vgV}B;t*Q|JdI5xL>u3yK770A7IpxR=dJ>sKL z{XBe$3wP+mB8-y+B3onFDg{-l106XBtfWW#{Q+6obcA&b)=> zo%hjXL_O-CZ_GwJ`2m_DZGJ7Hmldiln#C4AdaKEuowj z&Xw&VmZ9Q}}$B2)|D7L=q=JmU7@}nz+2{ggN zULRSk{^s6??v*k9fhQ2>x*JVS6Zf++!qyd3V{)jVKxx8)SJP z+QMTHkAB-g7Ga4>eyc}r^3j=HAm~Mtrh0qn9it;_@?z%+uT=ZU2fdZEmdG(e5MCHpjr^px z4BwYa;O;RtwgssccE^F0ISP#`S(t;iEh}QcpwnCLNYh9J_q0bG6S>B7$#WCIsXBd; z9KZzqDqs;9owz!e{}SceO(`<)wdNc{lMPvQbf>_Y%O}hH;Ks&kDMZlCMnJ??rD?om z4Dxjf@Y?1)>45AcU{bM>_XetG(re(pA#I1Kjg5pisq;ZeHXsxS&CSNjC6%* zO>`V&C}_m071}ADRJ&8wf?oUB#)X0 zFn2og+lHumtgifE^p^H((TQP`qjc!C+Bditcs$^vYBU1a4>G2o9XGt%qj9jQ)-ep1 zh~&I}@G@HK#%~+^;TxiNcCMeL?JD|wEK=R9ee!gpiZ?16Igr}z(}_aVKlT0QNDG@G zw&hhPY*J>^)g^N8q$R)%eD{c^&G3siu|Z|Emtnr4=lW;?87xgyy@Fwm{i5#rgTh~z zl(sJ^NfC87djtWWUe5faP}@`OX=tPEctA=?;F41-IkW%rDu8qetOIIF2P?1hl zCi*Cq^dko^nyA7hHIOpL(d`6uB0ccIWkkXO=|?=1;R5#&ve&1xI}uvAz(uxoy7nnD z2-}U$jqzU!t3x~}e~Fz9l^cu$HXJIA25DxBb&0T0<7@bK7}8H`)Xa(@Hn#4W7lF|E zP(&k+Or_9|%I(Hw{*&8NgmIzwD3Y;6#gQUyiRi{hC{+^i0BLyd-j*Pr`g{pclQ*Ck6`hDGfH9u@1b+je8V z1b_RAq|^2Sd9=Hv3hs7@K4SLO(jn}Xg!$bd?_2t9*4S63Un5KVO1g64+lk_|+u?>+yFe0rikSTYV z@qD>DoYUEF@LCrq`a?#>+D)Su}4 z8q7|}#e`d}N2Od{VGWu5)KELhU2GC@&){DcJt*E9QLN@W)5MB>b}Nw(hiq_N#~ieqw#qQ}Tk(u5Zr|d_HB5 zbd~AHmyXL$YmMG-L1}U>CY|YgTFTAL^@sJ74dnN6v~zXsI4vI5h$b;AsS0_}8}WMX z>O6`#<@>?c46A|yPtfgE- ztBw#s?vZN97~xHiJ0%48@-@;AmcMPF+CBeJ{ND8sIRNs@;Q@d!F$Io#m+)FU*k2Fz zA8^)Iin5v>KDoad5L4hwY;*Y{UjfnyV@#9as{rG(>Yan- z@47nw{;gCV%r~73)y*|HpBHotwEKVsb^c-@nY?eFOg- zmw%4Se`Lk}bH4n~nJ)&0D-5feY*)pZ&0KQS@2C(v=TkRC0-~dTQnYgyIE)qxgaHp> zU6eiwCHo3(e#$%YdTem!{C?Sfw2DPI@E8{yOBkuQ)OvM_Z&g%@JMzhr(%pX;I;!jB*KKtpb#MHuKWA{z`uK zkQH(%c0RqzX`5`;T?#6%4-Y=<7SZZ5d8Rooad9*#u?gU7e8A=XZ~}3JK9(mrKyN5T z^lW{`3IAsG9Oq}>K)3N{yDa@9H~2iLB8#at-%PkXfalpvIO0|jqkoQ(^4FmAkKHET z>*#lw)87gEzI$9pTC3jN+q$B^s6hrWszA_qZ|smXxJ!%onIr-q8=7h?PpgjmBDh(n zLbSmUO8ps6Rh_nZm(?fba!k7?_e4@=d9%E3=3oKzw5fICE$)H^`1i0!1h-vWgTv* z@jB_qM zdr+Dhnwy(57s8f|2%x3CZ8J)KF1Xe5q|8fzR^!4Zd_setwNT^+I%z3J9Q1D+?ERPa z(wl$VFb=DR@Qf0q13*3(GL{4=U_=KFE;IIcgjbw8OmBc*T3kIMPN%r*b5+aN`Fi#T z{lgzb=Rk%?#=1dDd{jk^M??ecHFIk>hd7BAf_NNFHx+sztj=H3lX%r$x8!kI=(T?J z+Z%UunRez})#ri%`%*brvsH4dJ;7+SVkA=CppV+jS;+`SgL@QW(S7j0>2VCWyVNu+A*8{l-ZLPqR^U#KLEQ z+bEloz5PYP=(e~hKrNJM;9VlCc)aeFjzWpaA$}oZ5%exWE#>pZNWokW}45`K({$o zn!MPxtMrC1VSrM;^~}J__+4+<%eQ<*!+tRVBL_a~m5Z{E?kl!gz+bPbV>UHn%_TYQ zyd5LXqJnZOjM^tREv@^8O^({~GOV)zlh+k+Mu6GHRLNKx7FsbO)4DXe%#8o8r;Aws z_BBe=sopA2Y=zos-uILi)gFtww2;f_hYL%e`-LM*z6=-(2UduQv zD-eq@`xLFz92Ksc_!*8RY#I-NL>1_Er(+~SE6Ojpn(Q?qmYlqR!zhboyW>`Dw>!r) zCOj)vKX`rmG2|-~jBbe1WAvcJ^xD>y1o|FtTB09)+l;eM%k%38FH zK~39m+u*!43%0T=469 zEk%&I*$+WLiK_2ij@IlRJA*{4_Y&iaCeaI;*1lSEOSbdlL1_7#sIl>)j~>y7usZjA z`IjfSaOjJZyF15HtL?K*5PM6v_R$(wn;OFvo{$wA@Ot1cHcda3kx3udcYTB;pTC3x zf#!}?lYU79RFvMCw*8J^^GCn$2DbHA`oQV$G62#(`Fqm?^51*d{hpq3)%)a(4ZB!6 zd4l}~!NvDF4tcLJ1-Sa3ALDOd%0EZrzkP0$&z#83$U#r@VVrU2Ut;hUCP1#&%}K-F|mS%Fr*aatJ0Nv+Y}t#2hFuc2c;&EpP*aK7j2 zg!vu2w;53J2lj?oFZo-(!c(6O#2~yJl@g$Z>;$iv-zxzw+EDqgy;$MiYYlzTb~8Qg z^%oyj{GzQ4!|$`P87)?YADT3`CH-7NP3S`!ykO)cl(m>7=)N!IH8Hx%9C&}n`Aq0%ia|1 zqmlzv7)>sd`h(FQT$`;X_bBIV>a?_UX+&i>UoDAe{k9>p6(RaZxBDknwr^ILFLA9e?j3C4Fv2{s{@Evhx>r1JR@m}w>9U`r(L z(Y(%GT->e=#qUq^-}NwxEX27#YW1*7cX_5(-u)t=8I;9%X&o-s1g4SPwm`p_g3G0c zafa>Uzxd!Ebkhnf3t)BpaOC~<^1BcI-?X}H%z%HxElZBzgT7Iz zZ6%23Ra#{y{z4!<^1jv3@5(y=-`KxKSEu`g5u+Lb@hhwzM`$PAeM(`mmTI!0jbZGD zBg)}LivAqy-;=!X+b?%ZVEJuU-i?694eWw>L{|E3k#A3_sA}bQ^Mz~qpo_}#RSq@# z@~?66yVnYw6q-?cM3J`yJDD4fdE-0dV2qbefoSDc0D6-Aym;LJc-@LqO%hvwl<(-@ z`Ieok0J0;0eDlD6agLjP}`i=cbf;EJN_Uc z-#DkzZYly6NQw2uh8=T`&*6!N7WCb z!-bo(tr)vxxYJUzyN14r8TEm>BaYuGS&Og0Fu+2Hw=9`XY}1K8brOJuU6aW>V}lbT zJk2|8wKY;qgGPx~FTSJZZEB^0(#t^;-dC?FXpAWWP#9;NtiF-?vf|c$nLkbfbM@!ZS+_vY8Sz9;Y^g#3)UlO`@6+(qJIpxt095676sJhf6#<8qA z8lh|C;3&m5Ia5Mjp-=49bk`EC;lESxt>cyf_Mn&gjOswzWyatnedOZl=I4~NV#A=P zLFwf;N=&C38c%8QE+kx3ifYklWr*Xx*j&M(%L+1(H^wCzFCBeRpPckImt=Gx8zQ}fdj3t3K~kxuWy7P)6<9L9as zKQJAG(*h6-wYFa?@2L*REPC3Dj!GTtQ9`0|*E{nlw06%uXy_o@tiZagmH3p?VKJZm zO(1pgw+%5Z6tjgD0 zIltDn9Q*>SD@n@3?q5OewvkLqS3|FUzoDIf2()j4Z-t`C zklCIWg@;m>&Bt$bCzwfVi|ayFN2DK}7@P2#%p zqpU@$<0H{e9X>l|uCNvsdOm0!G?cvh{>^$|TK-4jr?5JG*={R6jO$`F4+Y}*b;u21 z?!j^sU#}e;D_h9hRKD1(G6r@wO@0(>``kWwOG*va!O(VPH5&nL>VF8cPA85O(0c`q zp^gLl(p%aB)1e~79DP3^De|?%>-Ja&#H@J5^izk9_=5f{RcL99kzIa%TW}U<0k>QI zXKaroSjgzw<@AC5$w?<|pV@V<&O#e>)0@h&QwTy_ZNb5};jRZ1eVi+SIT+>=wu+3G zjICryyGx|u0>#eRb$qJ*p-Hs(n~eGC_Uh6%)l(49$ZZ7dk34KzlP-fk0f-U-p7RwX z{;dS{p*i}FP~?LuPV`~>*3CxE`EA{dKqQP|sajgKuuv^wc-+-C#4GF}`9bWL60ekq z_&E3HVSpg*mac zVfJ@L5jpOL**0$sv)%U=xdzSj^^JDQTvtG{@RzV4KKMCFi9Xv*rp7g2$?RM&fzI5SH~1kj4jQ$c~vZTCa%$G}k{)r^Pn;c~9X9UU@c^<`sNn16c0 z5{PHG$EkjcqQEuDX?6*CG7guG+O6mU(sdLYdWSwbF=$0e9xP6vqf9dGsNs%Zu^_z zuNlkF1Vj->IZkN1tNg)UN26olw%#F0>CV=^6Ni3WCRzvK7u@z!Lso6&9OTH=-!=pn zCsSi(>Q^LvAIY@c10+lC_CwaU+-L#pzO@{T(u%uugu(+^4@Z0b$~M1YN|^*T zG*=9k4~nlU_;G}ma0&A9>R`yxfd@AUn+va%hTab|`tf9_{rL{VuZ*$d8OO%Lu}sQT zms!XiiP!i})oDEdQx<>*^b>Y3drF6e)1CUs{N2l+ERdN4o3Y31p_+}X&m}?x`!c6M zX&^uCj6N|YLRB%R02efBtGr$R6Rhz|kJsiA4quGJPL=w;-qE{~QfllUDvB@CFD#gk zX~6PhDA>Nyei?HmNERvG*hO(PV7)zVX06llL7UsM&O=nB#v!E_n>d+!rI|z8v;=Av zrpddC?V_hwGGw%>=rN)0^i7jUCKLTJlJTM`L9anFfw4J(&wfPWBU*!e_h)+@ZCb+} zsmtTr%nSo&iE+YbIVkCHH`Oame3hi1tW81&3*M0K1jr+d{e!PrS$eMK;;28 z?&4k=P3`G$+#|6mjw~zOjuC{SjG*J1SR?f@3@!HBe(E$bPWQN z7XdV4l$ei^QNk*&XMR+0K~^F}r{pNC1|1Du1#4Ja0nGuj3U3}Xtu>2;rd5V%T7TK} zVrUPGFIpV2)RZ%!CtN#qk(2Pb;A=uM6|O!@#V3cAW($A)#2__<<(zeY$qCRm=M<%K zqx&Nut8--%PAB_pn*YO)^&d-I8*{wKR9M_O4+=SpVojo`ITc#jF{Xa`@hPJt^({?9 z?Txlq(mjbduMma2|M;qj|M29GAA={%_~z) zgA|UAjhjw~K-~P70?^#wHk5Rb0m=%KVtQ%0c)UZf(ju63_zblbiG-@3o%LyKsUHa= z&Wpps46{8__BUrZ=Q$lMi+@U_h&Yt6!xJ}(8_{WWe<0pZYM<_b1U@USWAnMbF?btX zK=#VVuQ=+Dlfz@K4vHseZD4gD{eQDf$6|hH(|fJ2Mz=%@7Suzl%imnHds2^%fxXIN z0#7!#z9BcjZmq8?{d?68Wp2VgnU*hQlw|4z9j-MED_Y#+FLiv#sy6t`rretx8stw2 z;}?{<2f^onIM%53C4;!C!e91&Qrt-CGKH>OlbsZ%>y5Z$Fq5Xo$qSpm;@c=8$c4Fa zA|At2F8TITq>V~{PlZxC+F zq@b-U(Q-Si110nExr!vg;g`&OH0cAAMVe2)(ZbQ68v`5qf!%Ql6;?%>KRLyJwB;rM zKW?wnl${R?Xj64bGb$^{eZ&?~2-`YTM=83%mmHC3r08qc>|9A@C4Bku@|u(26RCA^ zvT49AXL|NyQV19s@8?4GQ9Wa{QM0`WUTfDgl?3@hyW{3TtcG3{!8D}-=6${xOxMF^ zieP6rWjP{$N;qz-qHQ#%M4>N9N|ZdqC9@z$%x{Uv%_O{Mc4)xTfnQqkAud5tCPVHm zlwi?=sp7+c(4Nb4duRD2#kaS-Wrv))W6GafZ6djuMyzS}T!+WMsVz^xzvKufIP|RI z=7^C_8x;G6P8+b%vTy-k6L4Y(iB9vBc^zz>Q-yoVyGBqd_?Jp?H5r4`WUm`P=5M7R zedz#Rx-e)pO=@d>R6lHHlsywlfW}#8p>pXxXljG|QamtpmY}7V4(jylMPDM#Wt9fQ zPgn||0EQ{Ke|3LNYYkIRqN1e zMje!Iyj=UbtDCjBifg2yqI){{TcOx6$Ir(c_lIhPKV&1dv{7xzn1~^0Xr<;T8wh@` z_NyBEgI3*9%6`ql{A8GJo3r!ukh_7CV?g?Wc_3MA&uR@T*x0E6 z*^s3glc0fHD1}F5bnwZch>%F*q`^|3+IbPdeFrZ3DaRj_DN9(gc7$t2(*%8xEw>>^ z_&yoUyN#B1mP#*}OKtkT>d6qy0#Nz`1;PQEQFdi!@hN@B9sE8_Tu`QeK=D&Q;RB) zoIf|(zfqX^BmI;V-*F-WOt_?{GlI1?s&*iG&t4<5LMB5E2*<_)jzAF)QNR&qDv$#+ zbo#)Gqrd&d8>;smy6c1fOCW#Y(wvxK>8h*jUr!qIDN{O1v`_T zGn(p*hJ5{Lv}}*0q+=`>FgtCk(T3mJ?0wi@n$zW^anW1SC$-I3_H^osYSU|=Qu9%z z1f2rh38xdzu(kkPWC2>*K-~FNGh?&Uu8vv+wR4s6W|4wFK<>#`TnGzUQ7}4ILpJX0*4MR< zDR3}2I}}0G>Riz~4EXvUGi(bbPG?Mk-LNh8g}lqD{Vh$^W_)~P2&LNr{?%AyfrG6Z z-e+Wi{vCDK_cPj!YSF%=<$~+BdA!SU`1vY|6hIvS5@_}zjUnE7*Jr77>#aQEcJiS+ z0nO=kZKK=y2N|PlaUgF*9iD;;>5`ms&h+*4tXn=l-`&=>KRCvKz3LIVf~r`>&8#Rm zxK;(c!{u(d=$Wd0jd7&_kph6K zi39i>hwflaB^TYKXA8^NGb;_3FLUmaTM?n=qGowOR?jxOZ_&pK&`%j={8jJB*h{!g zR>w+i9>w6A^mn-nJC(0+(q#j;e*JaEGLDYMFW|_R#nW@l5{J5^?;518DG^zMUbFK0 zIOv`2b=yk%D^LWlUT*}Js(<4$Z0J6=Su?;FDH zZlxo2tkM^O=3u9l>U^nr7qEl!uwGet`vs9`uZV?s9& zhqs@H{wg%7F@fwRcs2G+QSJ_QKkaY9Vf$yS!QaUlq_N#7!LC-vL$rC-B3}XQ>m;4eGtpLqM@jD>re)cyw{tIQU=YF&i z|D|^aZQxTV)`^7j@797O+z%IWw}|mtteGZ&)=SCvN1L_5Qu}OO~E*Az{Uu`q<}(&Q%dp2 z3MY-kx1JDnC$C0&_M(mlSM%A7M_dwz>WsVtaYpAR%;MA!B{?C`sd}GM7|C8z%hO|+ zVMKa3E*KqSWt)SmZBBA82}*KN8KvjZOwAjD;*uzGdxfLas(5omyq3H4BMcRvR7*x7 zrrnEJUSt!C%f{}aGlM<)3kIBN%NV2{KbJrcUMZBBmnlAzX50!bL0m*=`?cKX%;u@$ zyIMz@Q~}v8&q=Jg=l$gSXrAjL`GSNmc90sh6V4`=zd-(Eosp*AMXd3TMG$9$^0mFS z+O>>@QqPU zR1xJ(YtSk5X)E-e*M0}-BQogjRq`LJ1pZDY4|-@;mh=(-(`T44QT{+4PDmQEE%({a@)^ z0&i29Jr`qv=;D)IY*w*GvLdlry9V#pRCF{r@We;ElqfFGsf$|9v3$-#|}Mk6tK4vtUb65SWHt(FXZ za@&I03`>-p-10B*mfZ!l$Q3>9p^6c*M{~{9Z(&1TmMGBfF5B+H6CWoG$J~$3@fxBQ zUYI_e^({!Xgkw6xSMtGyKfH}SQdYR!R)SRTuj##pvZ%@G*9r3m%S*K>ZXf;$cIE%X zHe27*=LK2u*EYXHtQx97VSkrq{fk%o-#pv@ldu0p3D|cCJ_SV*-1!C<2nV2MTB^fm z8@vRJUC||7Y~7K)O8yG@=OI`qhr!N=2zNhJ?czoEtON|US-b(SlVagNTEjn$cJdmf zJI87FQDLkKM57Sxog}d$eM(wG&`U3wf#6ceG-&v>2fcC8O%0|!Iodr&qiB-Zl$;TFou!q5 zbru!_@)5VCMPy)EkO#WdgTIzp$5i<;t2?%OZnP;ucNPbOz?s+A(iBjP*#dQRCHS^e z@IaF414$B50Ne7I;GCQsd6UnEL{pXw=ugFtQA%DgQ#m*41+XU@;q42F_=_;v#mCxb z)2>RDw_Nq(PWV-BUanYl}*dsg(mppPvVZ=L0DvnHMn zrGsoz8%>`#2Km&OCD? zboQqZzm0=$Uo6v(TZzAdKF&PB@>;HW{-GbwE6*`FQA`9XK`2{WrB${^ zG)|lknZ*f0WYu0W9m()1iE9cShI$yHzxA#1`mCp1gjo=3>xueTdOQ=O$6u%`M zW5Ys?uA=zub)Wdja?VRXTHo|DK~z$X^M~>80!1wA3^6}ANF}}w)CuS6CBm;p9fb?F|U zK`{!;g?Q);7#$}no{am4YTBT4qr{;$bzIJZqoFcY@yVoqY80fq7A8Roqzy#$2!oq-)PFDb1%;fO3v66eNReqn!d#e7dRqaz$y9lW*hJ2m6I z+B`v2-8`p!!8}aN=(?oZqBtX~Mhh~>2YPs)%W@t@6ZG$B7lcsDZ$k5>3}2++q~SE4rj=(9FR3g6Sd8W}W! zOS@KS;<#OAd3q3QT*8*cK2^7CUsa1KfgUMrvPcZn(3e94a#ZVO*Ywq27w=0r2!x#) zs@>SL#lnetxNNLvCD!cz%S3V`G|mG&qHRM?UQM|Eq`pVq9kiC`wF;s2n+y4!>RVQB zP;$B*&Bsg>$E%}JG6XD^7h)d+|N1-c6$-efUJ`V21imSR=4C{p49hi$ho3NUz@7yXvjX&FMR@6303Oz1G&v zOd=I{ntM{9P-7oO^A*2Ta@hsue*~TSzss!rpZeRM%;@YE@a$ywYhF9)Dvp*w-1QX) zp6qWLbBfx_5-yiPoeK2L#(~n?lia!3+&d#RW~hWg>$(CR*J-F@ufc%SILdm;(OPD= zxlq#B9TjWceg7eH>Mmw?NGS>rxZt}q0T>&Rq0~s8>Ugv0Z2mI5^ipa4rIy|&) zhJZ_)DTQ=uZ%ouVI|}ZSk3SB{X)d<-*3oOn^;NbfTagDJlf(o^UvS zBXc#u!)>sxe-_wXD{Jp3xd~$~*yc<*2W_AX(NA@qc+1NI3&7l!7shmpa$ z&oWCU;GY+U&F{@3Sfhw!Jb?AKlhp|*V+!|4j@bM03BY4C(gEW^0>nBj)^#0SQ8r$U zN)~P{JS$A655>W)K&n4jzP;MCGOy`gl783nNPW*!$7P54d%gzgWuA-M9^_56;YIAf$8O50<8(Hj`mTOlT(DZe82oJmw$Y8hGzgVZP}RJ{67pO4 zr&j&JXC**@erMVJ!63ak()2TT)240%?!lt6ZN;x?2|y0j{&z8cRrYWjZCH(-=rij0`{RgX% zngyFlRUGcsiGMcy=kWY!iL;C1@&MSYH z>-pP0w;l7i?c>1&{nu$FuQhkPAs~7UnP?qLvp0M5sX!P#I`~(<pS@^JPR8U|+Kj^SXfC3RIdW^l*y-!B8mwzYf!GE| zVy|VN5{Uzut_Cj5B^-(9r(0{(a^>MHdus@FWC<4C-gALMYa(ZRpzG(!6TD#V8n$VI zq09}C>2%;(dY?aM4V)Z%yRgGN#jMucy!Hyg&SeoK&jRh%geErD@x*yrBBpUA-cPM*G0omZfd9FW8s_M$W|nJ$|jZwocZ6oENY;vnE2y$#)4 z-nP}(Kwd*UMP>1}S^mLlmoK}M3z+HNK&4dG?}fgmE2i7-u#p9TvY_&(qf1-T>Bs9P zUK7d~sS)uw=va`Xzve^$^=i21K8&A-=~D`RaXv`SX(91tGox`2pEz(_CCy)D%ZK^YSggq?6rhr)GAIfW3E+ShQ5Zh6<&+KH@kxmb@q>^ ziPiPz{j-HZfz3G>E+8a1&Q+XHeJ+DWA}eGmHCC<2XUQE=7?$H@{5C$jpmb?(0-AuT z@1e;U3suKA-dzu?SAc{C`ddO85#Ai0bzu7EH?X}%Sp|q5ZGa16i6sm444>1_W2C1S zg_Cj@yY7*iS3?UmZ)s_nW@A_5a)W&H_Dpg`F=+>Y=C=JU$l*4Ve!X1GOYV1XgI$Ls z%d({f^X(0%td92a=K!L+^R2u00aG}gEv2xLP|wS1v_gM0JJq`_l><4hCG&+&)p~;@+lz6NTr<@sk6lgq zzRU{i=3Il$&;jiR;HiK?=KFaUM3U>fTEvnq zyLwE4e!zUWAUmzOhI`aDurfdfPNo%dP89G7dP1bG=_IGkuZ@XFTc&?`1*_>opQ{)rcEkHm}xX)p;wUbo}E3?Q;*q3>6$m{ zOs*l=5MEbY=>)X+P#?zt%Un8AEACfVUEgwa1~Q!<;k11Q;*fOz;X$IP-ZsZylZ8Is z_HG>Lj5zrSwKzQjX3z-x>V8&k|WcUEudwD$TLA1SZjNq<$Ne--$0rN z_>XyCA{jf7D^!%S_>TQa0jIiGhZ(}&Ye>yxlEJFInp{wnatc7{JV2;dL?_Y@Kagd5 zNy2eK%4*M|U;IQ1%`#d&_Q}FFuHm~#={#f5=Qh@CS17(touS5>1`g=lF^^P=4*l-v2R4bM19!k#5vr(DPlYJzs zcYvLDSLl~B5LgfUYS2VIlZ=mI1)^J3L*!^c3&~!i2B4rxF)3DQzTdG(w0N6ocY@Ej zKb=icvu;}ug25p4d=q<#*{|b=7ci3Et|=@lA6vYk-X0_o;-@;ZCagPutY14yLhdVw zYUn+S35SdMiKrs+C_ z^!@xA$UocN&_}fbpkh(qMr(F>waTD21u;nVXDtF+b)`{L(P%9}rzYHlo*40J*ZxPD`sC3h}g_kV1uVor8K_! zNdeR;*-^cr1<8fz`bL`aAl-xQL_;7S4k$*cq$NOcEEL?D{qxQJfVHCLe>BSbx9v{? z4pn;LxG@`I>=VEYEz>S@T{K$vmuk$PO|o=m{#djL76106P`PX7X|5E=(O9ABvp5B4rA=_{YX}c8uiMhX6BX zZzDn7+YR2}%l$fC-qIGtIrK{vp!;ex6QY5Y)ODe^L{FYC>iA0Pm|MeZrdQq~Swa#bWQn{S%Y-qM-o>B3tsZE6IO8p%m0$_0~xHlKjxh&WtI zDM+QgPWWllZb5?GXrz~x0y2U%2eP|n)dKYPkQ2LNy9CcwP)>_sw(|xX7TOeSxmYzH zgs7GbOd8}$SZT_4#eix*+0CVIP=2CkNS(W29dI*Wr}ny+Z+j%~KeHPe-i^4@P8u zJmYB5H1zF=|A;MY!iD0wt0=(HZ#2Wp%vs(yfg{883Z$G)t)gvX_8{A2t8^##MmnLL z74PZ?jU>3u6Ew43Lsok9nW|w+vr#{A+6VI^-p{n*JtdP*uwO2Op~CJ8f4ZQe4Jl9>Qb600IkN+*&2*u|v83Af+RFf@iEAM0(W0F^Q zk00Hz?#>A2j*` zZ?#>eJ%U1+*8#_=)U(pQpPFQX#g>32ibbP2bNS^XJ*6|Ys}_K_=4Pg(Qxb2x9B=!) z_W{hPvC=B-fHE<_XHvX2hYQY2?$({p z&lqs0fTw;`l2hCQPDAQlG4jKET(Q5g!QQ?8_wdFxI~OZrfS7E|x^hOx*q!ifqUK2n z6TTL#8jz1Vy_m;M3(e&KkYW7Xhvz+3?26Z9fb1PmZMqbnF|A4fqDzL8GIztlT?~J~ zC$RVqV_^WO?YsUTz3!4N_z|Do|G(Nh_pqe$weL5FnkjP}v$QmGW=cDF+Ihe!O*@R5 znVA)+m{NHbkxUUSmFXasrcN0;>Zp~b$wBh~C@4-ICy~qpm>`mvW*P{Zh;ryzJ=eSU zw)cHzx3B&5Uf1(n*Y*zuS!=OatabnH^}E0K=ZlP9-6^3(l;i~XxTqnOVo8Jd@F)bSt+|3PJ9z%a|Wy-hF!0GjL+ z?QFR$Zg?#ZHY@J@d%gb?hyPa|m(K#`)DB{&=>kYo&@^;Y>E zB`gGIjR)Vt(8);@}ot1K|xwR+9JD|2#cfzUq zKy1yXO}NA+o^Kw*A28;Q;}i;hFIf{p2j);AlN|%0JX^pz0*dQ@e@%MpDa+rHGJoh#MF*nK1}O`B{jX zN!e@BBoXp{e>vH!O-9c4=;@$n|QhpmL32t}uOy&*4r@YTsPWzY2Ok{W4Q-`WO_jlFU9+T_gk# zpzI4FffpWASPn%&Y4zYm^zQcefXP3b@cI-AMlfJp@JmvaDh8%nMFRUR!zC^QGydPV zlK)$uf3)!N0k~p$9QuGqzC8E-R~~x{yA{%@{E@{_L@DC^k{@S#|9AKDqipb% z%j$FZWy;GBF0-%qdIk7P0V=niqmg8; z2;GtD?;@a8tpbh7{G8f~NuklV?^c|J@BJ;qm?XG^?>p2`Fp<>_vy8E0E#L<%D>ioy ztoMGZ-p($OP_L2q*&md)t444tdSGDEL)%sli}Ww)i6W{;p#T`KPA{-A*LAq@-lKxd zNTuzVzx%29RlfnVElCI6f~vzUw3GcvyxeZsD1R1k8-Muno50s=OkeQ>`j6Mj5z(3t z6!IwXD+o0Uv5;ocs5o&Rx`75TF-jM6->u*hA8^DdOYGchT5<#CLBL}nxVFBIgJQ9^ ziO-3i!RI~YW!KI5ozv?( zjc9X{%_6=!PA_ilnRx97f$+tO7roY$-hzXBY6jPaWOR!oXgU6mS^v zc4*t2UUxkNGh&JWsQ8TOD8;~zn6Gxv`)`=*NwaWi7{lmu{obT3}@ z>3DpPZw!>;8v?4NF|bT%BY>V8(*TgO}|fh^`U zE(lR<8C{6fl$&%cC0Pj|e(9ux;vou4<`btDgVcrqh_sm=0KYrwxDsg7X&%+?b1+I6 z?O7KtvJQxV6g!C+mOG9gbX?zyOj#NZ*R?KtKozyP`waUdHo~k;=YV7m z{vhnntqP8>N-Dng7OuIeE*jTTCl`6uk(ND*i?nukB&hv%p__nqmF`qj4mH^^=? zYi-p_9vL4jaZNpW2%#No_?NIY@m|dHF@+;$x`|1Yf%_J=fIwOMW3 z&Ji5yhe5q&r!>QjxO!?Xg72Bz3UzL~c+)x@bioBX_L z62o1o>F8~0QexD3>3S|10cQv@PN^plrW_r6+&>Fw@7Y}&O6tcBba41)7A4~f6KZ4 zq45Y1xFelj8Ugt7(|}a}J^FRI)5xD?`k(CoXE^g`?fzhU{otwrv~$tQ9wgkCr?+Nk z#9ws|*EfLY@C;vCSqyz2qBi7t!!a7>39{m~IWb%+)=_&38k%O#4arz^p%zRzGyB36sHl2gj^ozrYj z*IFaI_UoJp$$b#LbF>6`4j}#SNrnL+e1lS`!@mA1kBt1cPk7_6qf3I z%ozC|05t~61_15HQyEuAt@bg9w$BHa{A3rvKgg;+o>Bdib<#6(YP&xAN}}zdZ+XL$Rl#YXOkKnpq!!0!l z5A6_0QdkzgYqG!G#V5}-ugE-rDlwAXk~;maiW=te-5g^5$_AttWLB4#o{So;f33EYj1*Lc zTbp+jW7Z5G+94)76`qi1kCE(0fqJpCkPoK3UNXMMz1Heb(*Z;VRmsg{+ZRCNj;DXr z*eSVAG{U;pq=0Z5+Dfm%WqPXCxK#Pe`I6f`W2u*4N{T9yY;!VVF2Nn|ZQezUdVvkj zBk|DA+2;tCLTVhfwB4U`aE_flPPIlN`>NLg-^hVhT#aOP3oZV>uA|s}~_*=(~@FLA#v%0W(g5Lugb}@?M;xk4@ zS|W2Vu(o07$lM8qz}#mLbFK%QJF;CrGyU%Kcvp|cXMi6p9O|0-RvtOZjuhjZI0+H- zpp09-elsV{F~O_nv%Z>U=t_IqFgo#qgN$BmDm-)#mSr{yfn*{c;cr;rO0pJMsvi2B zEhP0jT9(MRQoj8$<9m+!yF_zNqdDqaPN-)sr4hS`+EW=adzP9RCd0EPHb5W>JdbL{ z34SF?a>4@r0NPwR^>=iDp)@KLRYK%$m)$|{W+5K(9}w-dPyZka_O^=hZ=UT@;n_4* zu4Q8&aP+?o8*!8u3Xyul5$A{6!_gE2+rF2S?59}a#J^UP`kI2LWL-%I1GtSQP1x1GNJ9hu6+HuvJVlU7rjW4k#=2a`G?J_9n$@b?4KGuK;n^> z#gEhSTqet-!ovuC*tNu!$V&K@4cL&`d3Pg`>GeuV=R)E*;D?P|3Z!xwZz{i38ut#4 z*$CD#dgD<;HY+7Ozq)kU$w;bZG(Czx5;Adifg)1thiPTkX>pUsev(>tcxuN(&W<_B z4=cS#Sq3eT!Q2z(xrLbW$K%$Ro*&gAAz=Su?g+}g!!vN>g&pH3DL~geU;<5u8l^*YGZbGdtt79mH6lhuKSs!)=o|CbQIB8xn_URk zJ5QMmst(SIP4-KAuXb8UhKd>5%2j;~ort3EIPX?aZF_X(<*>h0-V;W?Y7yX7~YO}kg zd1`Q>&z0t))?M_z0L0&G?^aN<(v_}+srBYF2?r zTM2a8X>7w-fEmz*Y0QwKrO30b;L=+fo5=WXSXBb8X}XnIqRz}w1U|=i(YL~cA^fLP zzXPPb*6`7dsIM&@zXt%YGU{HNe8Wb43Y8p*C}?yF&9oF~apS|h|};6Up-`(Km-yEsCu`AN}7ds?eUKGa^%r8bw5;63R`W%kDV-k z3Qy1BG8+KJUfmO`8Zk1L$y0*4$z8>D3)dNT@|>|_gD)jX4|-%sUdS_|35~+3%WG_h zPyx0hID7*-vEP-9ws1sru?21l3&6w1Z?rI(=`YpI1#-hDV6C`B2n@Q%8k(!duT)uqo}C#IeunyO(l?nd^Q7w!$1 z7vROKFN&{^Y?pgIoH`Mvcl9W(A%$eQwZ0CU#H6EhdI8d90$i5RAY??!XA~RcNTEvA zK`k4mVwKK2O?-t}-PqCQrfmsEb_#c35*ysMFguLeyBh}xt=DE<^%QAHKWhv&N-J)= z;700<;CS^I6Rlh~dn}arK%LCJRcs>Am^;_zv~J$Oj{OolJ8L_O;B>W&@Zx}p%dJ)AXk#0V<^!SAV2oq188_Y z^CtkmDrW#GWQoM-3aTmYVCfJq_Xe!wk7S=g^_Oo5^1&mAS48{fGp+`b9^eHBUXjS% z=xEJ1pEC_sxr|kA49(9L?s^J-u#;Iap1f9GeaSLKw?`BQxG`^rcFOjiHe;@}Gn?vn z-5mtsGCUW&XIuvO_QI;PGw=dXFL4It(l{`5Kl}LX3u@ZnlSPPgYHR`ExReJ-_ulS{ z`CWetJs~D0A&7;FeMHPYrj)2T{ z;X?sU^TvlCYi%+?HAj(VnQjH>(;?#J0x z4hJqFjfNt(H}hzH0(C;wLSA#4%VI&qV6?O8&w%Lb_vIEvP(1ZR)9}9y$jfZhP|LyL z|5;1_oRq4(D#A?XzgsbB8-ah0`NMHg-B1P0Z#?{<)DE;b{p-&e7%ApDxoWjCVo>P} z0Jxr}miYK!!e4iFGiAQ{NULDjj7uOOR7RuR?DT$dEWur@qj32!1tMpUxiZB zjB47g-Ee@3SOlI$#x!~V;ktV!)?4F+k?s_YuWq_O{RrAhnyhu+Ty@6p&1~nkbBFop z#8x@(&I0r$GTA&n`xsT4S!IkZFiF1`q4j{*s^carD8Pp07(Y5#8Nnj?z;aQUP$l-4I(wI8(BBD^5}Se+jTQgHKfZ16E66 zj~X%sKio`{G%!r#$K0+$>5Q6ZTLxQc*Fey$+aVCzoW3>9P%tGU_Nqj@0+h#?XU3p< zy%g16UvRMg5v7S*m+g6ZB43*0k^yzO<;_gVMK4j*Gb4*RnQmIY$^eRA9bkNJE_tc8 zD_U|l_s@o4uXfeRYH%BDLJHqyo3ny6t*5=KbwF`l4){H0%#mY@7LE*#F69xN&DTfB`2Y(y39TJ?fnSpj_z| z8TacgPTvd}kLUhMQiwRF(Tbh|1uH8TIsc%mv-=`$!}y4SwDX42bBsU&jc8gJ8hMgy zXbt(EyUVgg+SZ7ht9Y%JBS=nxG*o!3;zh-k;d#{{HKl(N)F_n4#POVFqFycSd1lcw_=zoA6@!(U(%{O=U=a00WS! zIIW4e*|Xl=H2sh77aJQ+7vG{H0Lq_uM);u1B5+``syKOr{8jr!S8St}3BfQ$e1jP9 zq^xs$fho)^r7ZYX9^*taaL-F+&gs8J8p-L9K3j~|r2dz>HSoPp3U4D0p@&{zjR6Be z?}?B__9?9yuQL<&U8e0pa!Gh|^xmII>_uL8c04ZfyCBBr5klqvD$z=a%l&x`FOLJIx9X)W_FFKcP;Hbh8va zxMUdJ{Z`hmGN`Fe3$%_qgRSEqMAsx#p7GsOx2T+sLoS7GmzpC6K011QY;^hS>;G0e z{Iheae_ie?JNS!f15cf4{gK7sdbaspay=HJsg#dKRX>1JuVRrKWM~w2=ISL zTrzVsWFQW5-9jL@!|kHlQjy(s%0&zT`u0t}RoKM2>R0ORa#y&Jb!@_&g@me^jrp4@ z>rnNOy1Wt|6?>wqJZu=xq|q#O;vs@|x3ct#(DXNY5<&aRM+T)C)E$vobpMs>Plsw% zC{6pwa7!JiM|8XntP#8Px_2=4f=l(TMT{k1)s-X?wEHY|{!X&?ZD`)^$06BFE9M8p zHq7v|x}sPjX(F+=HfeHjoKI=8H@TU=r_^-wstO6(m0hVkI0uVxLlPU&)=*a~;(5O7 z_?w0slo?+=CQQidHqWMb-AK3#uaL+qPgIj;%l7en2mX zUL-uL342nKe;{xbuZ6-1FdIO<%|X+fDOk?oU#oI-w|;-3+2V~_suUdsA{iimu*2i1cPIk4D$u(cqfjBMCf8)(?9&@8Q>K_5Ag8 z+3LC{*|D7I1+cO$*o*+PeD@8$+iv+% z6IpYnrituH`T8>_CvG{E8;oMG7tztI{j8556qt1*g+Wc@I6+j;IUMgW(q2}!5QBW7+Uu11V)?-%RK=8g>1z1DZ%=kk~x za;-)byokf$Qo+Ht953gN^H9&0ecN63+IOOoY@ZfC^uUvc`v`96=xqoC8-XQjmMRee zAa#b71Xq>mVEr)6yN{(=s5IFG<>P`h)KABeLpk~?m%Dqab^Eu=sRCK%q4uXe14Cf8LnM?7nFae3ZPpig*D_^}o zp?@VN49O1|a_mEcR#1N^cy23#p7RqhUo2LM=sC2d9raJ0T8+MWlel$D#wuLW!vkj@__@IibGg=Q k=>;)vKZq7f`Sb}Ovm`vapSJ(6L&1Od?-}ml&?`+s(FxYFYJ?CD{ zoa>w4T;k8-G0?s%7S7dJ?ty{MI`U_m&0KeOIZrk?FH`{ja*s*=* zp51%)NbZ)Dl-he>pVZ#{dnG0J$?e;JP)1f(c8|3DAvu{t2V`Vrer-fzD^TZ~ZM(K@ z+a)6{!OvJ(Q7}T*OqU-+4{|n zU-gpM5)9m1_kXkP*xBs|F4^z6<9G15&cmHD-zUAU_^|7Q?hUr=-3Pt9>RXM+^ z_M2w^Yl=PkFKPB?#r~of4YX&g1kia~_k+wp!s}a4WS*0e?yFIbGdohZpKWYOq?#aE zI|I~ho|*hCX{mx!57Y3&=tfFjOEpR@defJvmwWf#Kw#47#EgY$>Iza|;=Ud(2IW0M zvu#Sy%NLZ{oW$WJQS2@;=!4d4G3evkJo@ewdR6MC7z9e+I2`T+?My5@*L*5ceL6yN zUC7(T()8@VTFHC3tB#a(H*P^8aI!8uYjH8z3hXHQrh)vh=3N49N2asGq3gb|BjXow zX8RvEjjJAFooU|->9?r(#fx1@J*kF`7nZ-4&Vu>L4nRyk-rGz0857t!r1kt&Uh?xq z7m&*tU7L$LLtpuUk+=ms4@DldK=}@~xy;|YB^<;c(IqZ)t&6g%Kma4fpfAdSV$f~67<3793#f7Hpp9AA?&5Uelt>Lk zGzW`8SHz$qC~r-92D;iI1}VLgyak4?;x;|RphSC7su*;Y!kGmT{vRlLL-%&Ev3rUz z6@i!VjZHZ05JDYVBv@bECS(6U{03WRO#aeSl~4C;rW zmr6lZ<_>&ip+UN6vy}THD{7Onejrds%n!!`JRAqOdclyL2<}C@& zSnRslCPG`dW_4e9D%FlZAmI~>g{xbYSIG03C!=T0m!sGm3~lEc;5)uTeiplw@)`CoV-)XoZ? zVWUV>;Vp$BB<;vrN2||^L9=4e=!z+l>p*@SGx;pq%Y6Jq*G<=m&grHH{4@T_Svl%F zO|S84G3Z%!_C)TsXg-dmyw@;A4BC?6?26=95PnqXex!}hG8p(alkV8(h!kAK51V`( z3TCXxPbSNWL7wIgxE{OV;PpQCbTiP*cRbzZ9GO1BzS`0aBgAx!mRF^Mew-L#rK zWXZUndwrm22Jl5fdK)b8p>IA44BQC(X1?7oqW0YV=y@lS7}P5T-6EnXSDegXFY9sa zV$5;TH#TSf1W_&L4&T1k-Ze@8NOzI*h|lUsLQ%$@4G}K<>Gf9*cEgS=og_RiT1yOC zz3dp%^sZdlFbDedwcwK>BE16k6z?IMFT!jipnTebSWIW$VPCo-}gJ1?fJrr5NJVMd6{Ef%a zp_E$IToCiDE0W<7Jio?n{jk|8kg;Sf#UKTmdCZW=j%)A2AzD)nXEd?HUR7Z42@e`Y z-H;(#ar`RF9(GVF*LgY9&UxZWR1-n)y*Xj2&K3uzW+OK4aLgLGxqA^NA)9!)NZFbS zTk}B0#xEAt^Q6?QV8)!A;tqjK+}IDBJ^ks;hQ}1_G?CD&HLm_kBwxQJqAZKT21CA| zXms1cgN$X%4a|%lA(MV-bN|z$LuNm|E&FZ^RZ1ZvFx*J!I{Y}-E^TGJDQBPE=hF4j zE-bnG=xXqFVWmL}({MrAi#dgsTcECd)=1oCa?}J8POv9KBlk~Q)7=ov*S$CQ>)%Yi zAPscWqZGJ->9@iczWSOv%SXR&m=(2nYhk|}uvF7Um6cIKLve6nGSG{AI@qnE<~cLr z0m8;GmgnC(3K+acp}ed{^7%gPfT#0yw)Z#h1!Gnsa=t(ONDQi)@nG?t&L!H64oe8!dOeY<-?86x*Vfp6hHVIz37G&y^ zeMNf10Ba{|F~$iGZi|rR{{Rds2_CSj-);H54269~`LD`^{tTFloj=D2d675t(>W7g zBXYVvtjqGOv2a>(Ba^1s#M~Fq)paVy1g?^-#Y~^7$w+k-ra9enz%kOkS`s3!2@dnF zQWWYg^PNH*&OJ$AvDI&mUL3kU+O2}-_G>Y@&KTy{>?(VJ1BPn{_1su}Fp19Sj=%BX z`Q6CZG+wu5J^Qw)hE0>`L-Knccu?yk-zkdhmIH@V0ZHX? z!z^T$PuUA%I!Tn#s#+@s@%5Lgp@ds$O!hT+?oR!tTy}C%KA=-223^KK5y=nclij|; zbti}ocD$QL%JWs@8{^CR7NOcmDw#|3s39bqBe}sE==D$JxNqNEp2E6hdo100@C4m> zh}gB#Gu?D+o%1|@-EJtApFd+`MG1&F1LwGp>^`0M(_i1`(T74tJ3b`3%M1?Dq8O|3 zN94i+DcO2ij{XKrbC;=AT!LKKXs+x+Vw8!aKm6oEvU%a61C5TZcMt%*tO^~U7B-<* zv*FKqwqnpO>CbgLzHBhraT|s?!JmVc_>NfyOuVlclx7@^0Ia*FDVI$hc1jtn(`Tt` z%YF^Yto~)n#`xvRiBXd4*TyQ(YD~9iE4od+*KA<|F#0OEi*LeTf+=Fqi-;kiKT^0I z3w9NQHg+)3Lk7S?-cd;j5@1r9BEe1V7c<{yr{qdys<``vD3iM#~1n+TS$c>Qv zf%55C>l3i*4{-gK&qGVPp+0oSDSj7tL3H|Sx=T~oeZ%DrKb4&CD`{O{@L9o#mFkUf zz$OrG%eVYclkNh3yTYo0D9zdw#a&OJ-Xn~V@`aOi^eoXvnX*(RF&0mc3t7x#^Wl<5 zg1X*V(0$1rqN+B&=>laP(D@m5BcfkD*NS$v|MENXy5DVN6Y{y+9-fU^42u(4k9v=kR{?Zw_RD+V%QLy#gds07Kyj7iht%sF-onw+6l@)36T!)Z=+;@Hw2 z>sPMba)n6G+DJlp<;247ew~eYg+^bJ7Db8k(xLm6)@N)s_8kl~&k|EbbFi z9bUv*?)JI2@WK6823JfJ)2vjX^+%FwF)hsbN-Tnx)l_TwIN(_z#Q!F~7yr>kKW%f} zBl=KhvIguOeYc@*snE%Bx{i5tn#^8m5&F(bDTGTd+`QC#kJH+y{toY?4=3I&tMoaD z?>O=u#_~w|ejFox3Dpk&gmMf)LD{F)~*AP~RIomC29AY~0ezT~M zBgeOn&-ZpN7|{F@eV&3ZaF$0d9VXA>x=54#y!baLTlB#h<-Ly0!O8Gk1=NM{nb$)B zbIgvNWW+;d16iO8GofX9>4^bwT#UbboBgz~ufpqFnV>w2{1$b` zFt}sf_AZNQ!KOwvm7Pc!q?)01iWv;mrev+9dVVHX43cv#5FH+vNb6g&kH<1x`aTHW zO+kv)6@8%@l+nVw&1^9!E&wf))jgkmg?XK?(lR>nF(o+t6xS9pl)*D|WJi)%aCvrp zZiZ@6_IfBz<7Qj*;L(mrdM8DD$#nhm`eNcVpF~Dpef`j(Q3!0mhMdh--EY#oSpk+w z_66g&h|ofvzdUd}!MQT1p1ZfOau~v#P5y)_j5|_L0YM-tLvw@lp&1!jlk|5%)eB`m zEvi4}z2T&P$o&Q%F6G#=x$1VbKTz^ZKX$P=*21F3!xOv#t`<~Oj4-u3AP#Y|Yt@M! za>XW&M#$(oi*7`Njc zng*d$#`JoW7C4tDEFYUnIl~-tbcXKqY7%Uy{R?|4+D_OSZ}%|2<$xTY>m#)b;(1Zn zko1Y%#+PJHApY%b!I2nic0H?`#HFofQtRtG>6AF3+@P6Y7vMa1PrS;+wPyP_q4yJ( z_6TmXTdJ$c->@f(DpV;M@X3Zjv{!`#J#hV*f_ch=wq-FW2=^kveO0%~Wi2sk^{9>W zK>4n!0i<37mi5 zuxsUv?8Y!>s5!{=*q|8XEx~bY&BQ$xfb&Cq2SRQ(rqvFl{b)T6BNZ$osHsR3dbIaF z2gb61qRn0zJ77ZH3yj>fS|7pe zAYg?LSYvr9x2ULa#OshuyN51UQ$qUsjH9De;QYkZLb^g{DV^x+#MN4$N4l&@e@3<# zaLH#l(^0Q77pEc&q)caF^Cn)WV)ZOxa(1JxBR2fc_flY$+hEs;YR4@`W5U)b|i-eta* z7D`)VP}Y+~T^R)lo97mW9rTfB(}iALN4$K*An80&U-ugu{xN}lgbU!=UgxRbgqI*x zV8^T@8&9sqXMi(;eQBK-`efdFZ^1d?8&d_;?V<5r)EU;UHQVpK6~>PnLB8zn#c@J` zVG3-R9kD}==v^X}(GIrpO+1<=`5{LyQyFwy(N6NKT=n{9_Vc7+j5<)Z2!FKn0?L}*haB#Q0u%UIon85y=mOg!-w)$Fr$A_)HypB`+0SrYh_&#DPqRx6 zTG&93xyJ2_Xl5lI`RXQ3VGDDrN;FeHkx;d6S&9~&UNvXya-3+CGjv7eti)ftiO35r z3%Lv#LVH9wbhEBuS=~>KC{Oe2Ii~3)@MO18N2o_uCk!#1fp9s0I6ofjK2pXE_npPC z8(XXWLbi@GpW@N6Zr@TP4Xh`a^J(nPoNqHHK2;*Fm?w|g$RQ|Re6uB!1&TUazP%Lp z;(Q`iZ3=aEB0{~zFl@3qOSQu$4_ciaa*B{C>r4gztZs#w(i)bZf^Us2gihYcroay@^F~tqn4`U7#Csfx$!E9VbHK<0l_on)3V5OgXn0@}H(@z9a9L2_kJEoX& zh%RhB816Z=yI(IWQ-4JfZp9=q-nj;MxhN}{1Q+tQvOZz4mnJ%LWq9x_k1r)!^{z@E zg*d@8ue#k1cNrWshetU3%BER(0#+^yKweO*1<(_}VJhXNHK+$+ij@gQawDU-&3|dFc`_cTD>L`h#wOa`cZr zo5-i+6SD?b_W~Z&t|npPCgc*Xz}vMA(j}?A+0{jyR1Tjir%V{eB3PXCmR`;&mS@!% zV4^}SEh;)VF)TYAxEQ@T~6}pdPim=rPHdpDOV|B29S0piAK@Xx;|xQ>0O51 zy%Bgp@*4S8&9hU)1PIZl(qp9ba}ql0-A0tpK$?ubIViNd*)Gv-8&6d$Mfv$!OP}hX z0=Odn{-T!Kcs>}mrR;3Q1~`4ClxDV|$ew75ROmCgRKzRl<62M&Apw%ECGJU&eGV9W zUdRYqOt1xD948DneX#D-dfyqJx%u4IW$Os8?2ZtX)mEhJ!jtw zUkf5&%Fz_>c)KRb0fF^+cig7kGc7$}RrHq2q&B~SsJfU{Z5a5_?=R4sq-vIE}%_8n2JLjObn=fQv-ls9(XMh~46gVudO6`35Qtf<#| z9F7gzCD43R1xt7P9^cxQ5a=|Ol zd0EH~A_9I*?FWO1?BZFu%_r(ATE&wN9*h;ONybWKfY4HOpg?8odXcL_ z|7Kd$bc`=PeV{O5Y~)kF#sgiw_{p*d7T72TmyE2#kS1~P0kZ*ZVGy9Bg{IeOn)&g` z(4}XQr$)S$42-rBqXTV2TJIcPt%M;YO(O3lt6@|UC@WoL1obO;oP3h^3_b>ZR-3w3 z61zElov)M;m_K5-?#euS&m$Thm@=7Zu#y2_FLYAyXx{0Jqu|75*Ee;Lyst#leS>=U zs4AH%q=8J?>zsx@1USLk5LJ=8I8@;HGArAXWtKScuq?&4x9$W+*H)v-leIVj0D2_w zb&5u~uPoPhgA5j}g%J$)tI^S|UDLtMtB`ABTYXZJzA7>Doz9-i-*Y?Gn8F|Gi4~HI53e=L#`nXm1}4S=IDn{&?*`_ z2z7)NlV(o|EO>HW#dkxn4vFsj__YH_n3=?9WFKTe1A5@gC=0?;NC7X>WYO5vWeIa=OnAmKp`?i0ARMYQC7Ey`y;nuGcx$PJg+bK0c$MGt3Z6lvB_XE}D zZJwr#{dMO<-z3`kY#6W4@+j<;abw;o(ZTMWv*a^IG4#k*VOB{SY~4?JSkOhk2cg$B z0l13`xpF%IO|qEpDO*tieVVJxBVYH!tJC^6dVJHY3ZC&(lMNi8A22=b9GUMvmj?PB z8HtO8s1iSZD)Tt~ph++lv;I+-i=A{n&a-EIWVk-4$H}oPAjgNS#Pk z1Fn)J3+Y-pa&PFDueoof-`h%p@dH>BiGA+~NuG%d3aRs)MJTEE{7DaF{ux{zW%#H$ zN`%b(lNx{LyiI~UEE|V|3hE0MJhs* zaM)lGS;yooxnbE}W5o-uuTZu<`pQR8))iFL8RU65|Mb$Aq@qN36bY3o(xfy)?=Nh9 zE2G>{(d4J3E_>LbP%V<{W{$2a8P7jj(49hYM-C?f__>7Xx2Wrbb=IaCn(5Dcli{lU z)3C!}j|_4|d(Fl@7gpr?$Y$n27EcU9b6m$-wgY0V@*Fr zERv5oLpp-tIHx?kU)#I7x9u3ENpCt_i-{hWo#dcog;j|$X5(AXN4?LCbEJpOh$dr0 zW?p;}s{2$IgUBOu`QXhh)I6iTL#Z-uFkf%ZC zpNm1eQHM+DO!$z(bEEjt;!=|Nk!9`C7Am4t77UKSbk-U3HAJ6t4)&DkK~AuOQj2HO z_F?M@_^_8*cj6s=O2i;bT%W|HbF=5xux~ggb+XUj%zQpj*tKM_S*fHT?as1`F#}Ox z7X5?2-8te&Kyg+Q3}}@tZ+*%Qn3c6eL%#r`i!xaRMx(a$W1d4EkwX<_ngP7 zHl5pWD0I8Qebbs|nnoP8qs|nb<|^fbcDtO$Wg9*En<&;^*A}H6m7J8%9s1O@D8}=6 z75U7-))c+)bhy@%h0C-a?%{ga$z0jg#8?3s0asc`YJD^A=xRZb zi!PvsL6meMFDf75YWygLe= z%HtfALUeag-jycTus+((3Ji_P6o* z(cxL%Uy}8ab)}tkTVs9m$C%P{`-VTDU(FDEoR83&{k@LF?chI5pA>_B8eEW($Ud3q zt~o-~cK01RczHeaq_?BLxv&MfC}^E5Syh&rF_kC4At$a@^v|M)@tMIQo!3tpKiXFZEaW=< zXd0PLnV-wWBk0p;3UQLb#kjk+Q4rkt%MTjejgP}nI9DXT?)tFqjmgO|0fYuz|(FVRelGL zKXs{JkDdl8aMDC+?aLgE&v;NQq|x$z zI=hCH{3`l3Z4lyURWUGGg;-x^&@FYpXO7^LAJ$S7?aw&pAT~(CzK5d0M!h~ zqptJpMMe#Z@D=KtE{ez!ZRRfkG>z^mH1k&fq(tqtbj0hKps>JKXW(ADFYT`QB;9JE(M*=iv@k8^xR+DsW0p8a0ZDLbzNuzX>=(aw(LqAANicuc&Vy?tUvXR zvquGyuP1--r+$qPu$h5boX-^D>%R?vcijg}dU{rw&%>j=U4V}|V8-S6P-P?>`4&T>NPr(kiL+df#|ebM zW&kaRwZAB9_U9TDnjB?iODMIx~|?c)xtlb-Qp=vg!gh#YzU@h#vP z)dQz~?mF1sr}3hV6wJp$qP|o6tLNT-`R3HpGykBM|MJ<;e{TFAkBk3**7#-UY+1v> zW|+n%m2}(+U~Dx(I3_eFbDs@dj0bn zfHU%Mwp00^+0OrwYvgYnU)gqnIVYAU{eepx8Ws(i7`qb*3}Sbt#r{?$G|kgxN<*Zh zIsL@s7`8mk0fBUgvC8Q!+9;dy^8T_>Eugy?h!$sqw)V5VKa`(h?MPfisYc|%jIJf? zX9qin%F$@nf~RTch60kUPY%*B@6a(k%t>^dp;2&Swqujwrt)y0`WaD+-si(D z$}m{U3g$*GICId=iTgryCK?@#5)xn2XI)4TUwNbc%kbR|B^~^!m?eXy#r5L$=FYHD z^TV=V!k8^wcEd=#oekkG`3T~{E)8VeSKoWi1`#<1z;<~waY_tAE)Iwe@~HB$IQ&ye z)nMNl{26_$1?&{7<39POB{ve8aZj|gS$qE8k=D{*Wj#BNP8brMVQW9t?||-U6BZVP z8t$-y<3?>@DsVl2b;N9Ml>u`-Xvls{``HVW`Jn!Ze3xeeW6u|5VQ*is!lNMz3gx4* zWu&a&Q3U>-lfTkKbD0~be{~T5*{y~J(4@zUl({VQSACySCP48wofEwyiXLA)5|y|K zC{z9b{k6J$ss^sS)&=m@0T8hl0qMH+6VURD-)MCkNEW&Yqlk{>W^N38c~*DSAMPbv z06(EIb8?9YFnAdqN8-4Y>FfyW+n-X#5GO3z3+pj5wwm!Px2*$CMs(?3>+GvC)M=5(8Q?K=@dfpes*rR$w8*x%b$4myI69~~OGsPP7VQiPT6In|1QVAv-1}Ae z`huj0N}9<+LIh>bh-el*^~RUHDKG10u{TaX|T9>W&t0L zF%G#GCHtsRkICU29*IOy7(f7LI$E&5AjGlZY=~#eNc{10U*i9iN<>xa>T03KY;RA+ zv&|Z5`a&#i#D|#dLbvwtArhCbRK8~i!D<-<%@thx}l!aC)ySQw;EXO z_z1mmb|zTa=Pf=>WUaZNIMv>&2#|O6<)6Cz0K5og&XP1c5OVV}TiBKTJhO46RljSJ zWcQ`)$Bm3z4iI1zZ7Dat$V;3s6uloH*km8KHCTeAH1-d>(yS_V)vc9jauzAPf<&Pk zvz8fF54UGeCcMGUD3pzhh70jn-}cX79m@*hUI6U$xXh+oM7Ov1-aw!1c0hO@vF;l$ zNVWiAlEoyl%fXAG^@%MP?(rt99`nC%b?jZc;9{%STs-4wEmgblqNO9^)Z!GHA!x@4 z7brow1|hAtua1Vz*} zPGM^do+!WkRqwsKhpRD~=7H?E@9PK4^;OgD39(Z% z+P(pE@R<|%n(QmNI#xqGqS~TpotdTIKPOVC;3zn0vPhEZA91f~e*Dj(1zXmRcsIc% z&S)HhrAWeBT9OECaz4`i!LvKZ4f3iExdx%SM=Hzagi9GN__d{{Ihsx;P?n~et zhM^lSZABvuUHVJOvHga>DDD?7rB5PBD~{nMDE(f7joChzt|Ki+AIut)-d4H?B$fG) z&?Hfzx-NugIKH$GZskdg{wW29#SZ7|!EJhl~mQCzNG) zlt$+MQ@y_UIjJ4y6=^$&r0Sp=62r5~YSeRbdA*FH-OkhzP;>n`UQpux8HIh{8N?NQ z{@5O1T>k^}8he+VfpCRB=J%(A1 zKKNKlXlmF^0{U^S+?(T8DYIez!g!5S2m_Io)WFU{u;;e|`U9rn=tgaS-S>UxVmQ9! z+UYog@Pv5|-#(oU#g6mJNE%4QEkIu?V`RS1yeK5M{-OtBv_#QX8;o9Z+A<~$a%fdVh z=<>Q%nb~>`f8g`Ree91f%FOCk0KU`JTpfT1%G3-1u;fG_g&|0!T1oX?Ww_oT_S_r1 zkJOx}$wIZv6Rf_(b>0{BX$>-jv#a4i7*av0$mAL^s4GzU>6Hlbc~>?138x_>HxdkO z&1{Vu%lS#K-g9s%=Tm}&qs6mtutZw-D_%Latsf^9f2^HJGB{Z17~)e znGTC-LGN>hRV&;oVg$|*vKUH;D?_%Kv-&o7YC5Yc7Lzhk(*B#zI^TA!0K~o^F(}j}w*+;nhoETu)Q+|=m13U)w&{S; zjGt<$jAx#gUldTZE~q0L$TC&z$ZXZZU(y}7IJ>Z%mrlWo(k%F$odbr7dzMd&PKMB! zjl@Xpnsf(NGOl(Jo=rqB6}6 zulM3(k8se#URk&TX%t-Ond)8Z(t*4Cdth%{2Znyr=^BuXkV>?xrCib8$4xkkx;&9> zc)On6JoP%CIVZ1Rl3AYp#?rH=@nGw5%w@eucNE145y2cSvpqGT{It$_>}Yta5g1#h z;Q8|%$s*!XQIb1t=y@krIEd61GMLSfLgEL+E)9ZJQ5|*daG)aiZO$h5da`R`)po`g z3{fp!a4B*PQpXOYL=QO*)EVYC`=i4V+c2=?k%*kWJIagHtVu+n`$ptaXDPy5$j%YWDD6ukzhMlLY!*&Vuc6*6N5mWj1mt(oYxaE^jO7#dj%MQ0fvp zVTOi3GYzKr??h^ABfGylv!WAvO%aG}ZU|gM^L@g6vBGENume_uM9-1iCp6-!twwb^ z-Z=q?);>O=#U=8SoB?fbZsCYe_&fz-7)k}?8Y(N;gB1RPH96|hJ59wB=iu*&bs$kwoj!5WMy7p8kll94&6<-==4a# zLBWMmXCH?!)TYr$X)C%&SbH8dX+DW+=9ADatG9Q5cY5l@gB<4x<;Oc(dU(eC%iw7B z`0ato=t}2MKey(0hhGm*0lSgBL|)e7;5}#+cQ^pxTgt76@oB`^d$h`k^x)43nzc%@ zjBwhxZpMtnB`<^fy7 zLk}4W{ZoFWBn9i=0Gd|2_|G;d|6hClCNy|^E=-@=6t`zzzKu#e7wnkkvyv+j*&{l@ zOJVK8BaSUjkini$AeSbw4Y<)|5mG3L2*{N~e@P2z6}=zI-BiV}D8opR4KW(MFee6C zd%lwHQTQ#%31`U5#R2qb+NdHsCk_=5ZwXCC>)v!~0=5y2N;3-j$d@0RlN^6M7Od+& z`AXUNb#!qpHWmoFe6^qevPKkBhiLzZP@zrW!8Y@0;gswPsnt?HvT7nsBmATM7m1uXFVG z9&~U#JL4$t67_bV+>;oqQyl|uFYWX79wwsNDO$`d1SkwF+W-{#W|26jhHFN)1bpBY-{xvoJH8uWaHU6%x z|DQ26k|55(TT@G+zuS8Ik8z$@zXJ>O6&)N$69nrumy!E5Md$(aAGygmf~f{@%7O+b zQTPyNwrbJp|EaaV?|K;(s7vNf%O7tTV_^A1B12g6Rq=x3X46}-hjs!?XWV#MhZrQus~wy|?>Y5E|9rnl&Lr}L zXfUpgf9mr+vNO?_L3*&bk&a{&A{@=-w|+5J{)X2+&BDd#pAl#W7TS8T=y4sb{i_{L z_XkJq&O0FS>iSJx$14|s%-ODYkqcp#PfX;?okVT5>CuaN!+Apmy=yjs709)Cm3Xr> z3R@F+e}rSLz4`kM=m+Rd6K6K%N`(3kaDE5!u(z@oxxVK9$c#cQZ6U!ES^s5)i0qoI z#NI=l|c^lgNt<_2XY+a z6hQiU5Qd%qc~?=iwOP=pM{D-U!420ANw+tAW0=c&y7Du+Q?CUu<_1o-FGpxe8Le}E zxa2f)SyBguUB;kD6q4J!LYRZg$=vO^#m*n@c=+a>LoDPwO~j*XODRFP-X8%ZsNg?4 zZs@oF|8}y3JhsWG9rPj!>RXd^HU=}Lf zA^iGj@(nmO{eht4-{@tsoaj{kHKxEwh^2T-NjmqLgil;h8nl(>Yqd6p*|uQ!W|8a| zh8@Zs(!hC0qQODY!jDi%GlbA)eCRz^ZprzbXoO=ID}Z91XDUNl9Yo#L71hKf=ge7{ zp&neJO<^NE*cU-?*t)6{%ez8fuN^Qon5Aq(>cKNijEw{qS&d<9X^Ex6()JB^0?g1- zFWdnhG*U)N2WWU$=x)&{S(nUijYj!lQJU5Lq9ZKgE`0QK~%?6Pk3~cuN<>u znXYG{yl=T7syFo;aZf^@}G&D%`46`ndtVesoPg8 z+6a+QAcz>wC$K2{DBifIwNXAx3H3cfM9b#b@)mku+!annJ;3F_L6;MnW0k*9hDj%H8v;}HJMa_dGO&wJIX zZYyTUx>&8hr62LDGVBu4rdOAEr{#)qC@uEVQ$6i1Gh#NEVq)L|R2{AFNaJ z_=qbb6s?EgQ!>&g9gvItlP{EEy->wDG_h;C5xp06xr)Pmgk6O`HI*5_3(p61ARsOc zYdCO0^$Um&E*;3afE`Ys1Op*C+^fhAG$8QG6d+1=II&f)sEcfAm+B_b+5j(M7#94ZVLQL$8R6=Q(>ab&;=rehifpF$R6p>O> z>j1?^q{%_b-fqNk3n0w-K|>(151|d5*5Sa5!GyraJ0p+rLfI?VA%;yZb+wgr2EK#1 zpzJ}UwU1t@Tpa1Bjgwd-$S&`QH}mCpotiLjeIzg&&pxpHX^?rXHUYXnB>95Uv{S;e zeo9bImE~mHBnCwVg{_=t4LLIA%!@<{o0$_yC51?{QSFd+Us)S)uCR zVyvgNmsD01xrpWR$^4hnB_+6@P1Pf*IhsA>;~`6pKwhUv>C(g@Qzh72AGgyaGL_s& z`~;*xA%;4TI(DkgMD%;~Xmi$jJ0~Wg4(Txtj5{E#zEI~x6iHQao){_=m1>|$JUyYR z>F~&vAS&2_-^nzf=Dm;bH%IA?GifU@CFt4b;u z89HFmH7V!>Flk2sR6xCo=r>0}#}esPEea!}5|lyFA=j^?azlb!EP)h z95IDE`$EE}GcR7|vv=WvSv!$3#Hd6A9wdv1H?qE>%HSF^ELZ7fN$(L0| zuWIy#CTnkM*5bM*ONpWj%M8wrg)WxzE?#g`kPD^pa|%m4y3W3k7k;BhlVGPq^Po7u zVc10j+$b2Cnu?|Y{S;>&68s*TXnLF%KS5FEDYoPssAi|fp^gvw+a8U6y*>wz3ooHr z(axI?Hqw0uH{c^a|2F}rb(GdzJhLG;T4A^sWk1m+Z$-_A^f}x5)q4M$Q{x!K=82wG3aZtH=4_#@G{NDYa+J-T#;)%V$jkAnxl`Z+&WT%lL8XT$9d8K0&EOy zLn-lMP*X3k7BsSzMX^CK0BcwgIA$R~?P7T9KW|d+!AI!_d3A;&#h5WOq0EA`NMtAm z9kdh}O20>Mo}noV1xV4!1ZRoGTaM9UP~(q+9RLoU4bYUCEkN2Npum3+rQ_QR?Xheo z&sX^;;BdH`cr!kaDX>-k8{N*|+wsoc4UqJ2<^EeqG=Ac$sW+nk=2PgQ|AQ!#FO(+D zLy-Soa?(FKCdGuMAjZ}A~1{8P@--wO%+AK)>40^FTMMfBzraFCYwU|^yk z1ctb@c_06Qs)(`Eeo0Xi&H&x+@#c(%8MWGV6mx6*zQ={8?y6$7hIP^i2(Cvj6sLTY#Yw$Sh6X1lQ zc_4#e0m&Kx@@jGF%5Oi#-x-%uXtm}9y(tJuzcjQ|cexFkrIqUv$)?MrVe&-Jsi?Fiyv!A{9^L&TTkgXpPQpoaX zRjpg@${gv}w`P#&?IdCQWRJC066P#WD_E*YA9ABAG-u?9wlEbBNTcX(^&qA2NR($y zYe#&QMOz1%l)Z{9lE(BIA8P=)Oe+GEg(p&;C-syqSN($85nC^N7a-BW^D%q-1z~jh zdCX{ap524a#=5$$ohx#0v@j`rc}}6V#Lnm{`^;Z?PGWs;oq6OZ0Cwt#d3(U=6)P{G zFWwm1KW`>4g<2Of4TTh-X%t2zM^oK`!|NB=*P4t-QG1yH*CO`|=1{;Gez6QxAZRS) zNjp4(OOZX9{w{f_mw2~5rziEuPZ>W*T1jv5OB(6LhsRi_vfz(H*&|&4LBhdKj1CJ{ zfPaP7S9d!Z$ikDwD@$IJB>nqJV|-M}n2op-pBkT{m4AexSJzorM}gQ9tLkjq)5_Nh zLsl{Gszrqb&*gWzjh@nqpKy7LHxnO_4jKpM_fD4DHFqd9JH`_xb&uBHI6FK@Fvx1P zWi!{zeI6>kRGGGfkGq_-hN;JivaD`hU4XCQOR^gUM(at)YmSVBY>&sC^%!$AGFQQr zCQQCDG?la#@F6+EH3y4AvMyt;k0QNp)&iDT6ba6m?utK1aXX~may|xlW(+r*Tk<0H z7j9=Jl=d^QF&JQNp6aJ-R%Jedd@=6s4GqqC_aJ9J0%QHJ+1D?C4?EPW{k33uqy?AyF+Z$XGRU>p*1jiW*Q!8-~X#k=32K2qqtr~QrPbYM|O zeTYuUf6gv!tL0Sv+Dnf@c>0Av)S@EF90zu`KgQsP`c*SL3g1TCxH=`Hdt|3@l1%e9 z7kdTfDR}sKHV`Vb6Hd{7&-JUN-jcw*UXQtw#N_mrqXP5!jB~QP^gR_t137U;8a8>U z59teKA}8r|LwHZwQv~@Q`XE$xY_e~wwdgVN{tET-vA$5`v8gG~BgF=>Zbul0Tsbtz z6hY|$gUB0On0ha1KR3(=@@(x+&J@txwSg+zT$J^fPS+){<$OZYbJ-E^2`3A;?rR*Oe#YnOx=PfP>Lr??#B%cf zI&YtC2UV&O<208Wf-Pd?ivafLaklSVVE>^e6|VqaHcXALmSqDvFsa24_0S60%uGB~ z_k0wy4<9WD6RT}7`f(R&=r!(p)-~#RNm8ZvnB$q+q!DUN^-{vo-sVXA!8%GXWU2{B zdWIr9+VL+Gdn4A~OWph==|rNe`Le{ROz<$cyC~*tGt4toU19W%cW}08AcKTn5lItN zR+ASh@HQiLy)4F;bmFIDg5e- zbs1!+FCa0~)}pyVSkF9PL8wozcQJqkE5=#Rupz5#7YK(Ez4)t1MQy~*%`(C1)5sQ1 zt;o`^7y_yTR04@EnZskEGNffxFQC0q|JOV4k*PKgFU$C8IBsD>L-FiUGa1qJLtg@m zOGc_sdmtL1fk80na-nlm6Y=38VYcVUii(5LWqhY+Xt$QtUIQeaHYc%i=9b4hFL1j-y2AUdXLqpZ8 z!ynhy3#|jl6E=zzLX*D*FF}K}I;-HIg@Mr8K)=H8Lm_OQy#$kR%cP=GamC5M|DD?$ znTz0A8bzeddavz|wVDg$b%ADeUSjlc}%7~Xx%NfD{3VqYYJSaU6}(EKYAJs zNl|Y^*EcpbTV#>I3Z!V+w)jmW!(i#()?>fPAmhfSNkmKW<5i*+BCI_pk-It*F+oF0 ztNpi)Rg{d@HJcw>@di)QNA*K{lZw2=w5()&)c$MDBr^QI9SUS(E1)`iu{xxMf;VnNoX6ySUmVjXnp0f%IqDhrJ`U!8DYB)8h;p2L_)UYBon82vYc| z0Lk{UjIWyWHZlnl<90`dfjnw%D^Y#=JH5EeVTC7**GzMNhKY_N>YO=jpr*3v%&t_A zuA)aak%Cnzg8QA`hbp+X^7Rs%E6ucJp7WisGF-bfZe`N`tQ0ae>TVnz_CdIK{UMC0 z9nYtY{Lz`YMcMS1S@WIsQAUy-_jmjP1+|%Mi;XlAa)*oHld%xen6d!bYS!24UR)KH zg9OlE(ZIyx$LJwvO)7%V!;)E%wI~O&c)GljKp4uVzF&#+#=TNpId*Qh=v1s(eF_Kw z6!hahUMC%E`qq6Yi$+a%rwwGq^+~YqQYUf4otlMByRF}DPD%HjZ%^y!#Gl)QCQVcq z14U_Xc>-B+iafTdKWPc#aQ)XyrN_l9zdV*f#=OR(Z7E&$Z&oZ;IMN%|D?l3wGx4EB zJi^>F=M-5%D-?^C?FB~r7W^a0lIS!YI?tpTHIfzL2_7DnpUQH2;$o$6tI+#5-P9 zI7GqrItEw0*Mq%i7MmjC6wKf&<0OLr>Y`3#4ug9m%waYbT2%we95+B*Hh4I)(R-B+~_l}z4MdRO(=yYW?*wgx2S8SYdh<(>PnI=OR-l$SV%53*2YlNg#>=O(BJ z1z25yY3UQ98~-8u5L2(O_B^G)yBZ5Q4~3)UP=vwAfwc|ob#=bNWWhnJTeN*aTIwh+ z%Le@z{N^beh*9%{vQK7Nwpj{>=O@#O zAQ~AY9O)sECUl%vw!j5i&6Qr2C*aR>suT>>;>ilD%bi~nU1a)iu{(N8znnZQvEIDf zZk16!4foMu2aI&Z#?RIXu_2+k!Bc~DJ5WtwIeq>ft)(8%5gg8~;K@$$)miP6SK1`! zd}X&iF9|$pfu&`;3gP+zK=BiSw1mbUBM!W}bYM+86-Ysu7kuuF>k~deyiiycmd#r5 z_c~_hooyQiD*_9xKLsW`ELD~kb5;L7N3gr(U;7NuhsOAM?k%_>`MYGj749ycS)U1fl{~;$GrzE-^sv$@?4&TjPR>cO#2g8Cg;M!xa5&qAjC|c>7DtD3v~Hk z>I%RI;p>Xqoj?p`x5s~8DVW%-=_!+&mLC0c+f&ibbL;wBX6L<#w2ikp9K8}+OG zc>R+P$2lix?MUXs(hV(HEptqD+2g=Kj*>!8Hn14LNZ6~egHiiY`(<$P3Zql~n+;F29%@c*9qe8XQy-43!lFavWP65+Emh!8*lI)~Ru1_+D%Lw`{A;KmRP+r| zzg3LMQtm)rt@4l1gLQlDj*R^*4l-`70~T#hc70<*2)57@pxwR5WAT$MUdinTJ?-mw6rU#DcX1{5U`hu4_hk2jVFS^R|mEx8xZ!hY8o5oB|fw8!Bfa zmUTM|r8=$X-KA@TigSSh*2QyR4qEFP7Mx%OQCb=@X)k);__YRVASyR%6QiZ(#z*3V za4DFU)aIyuJMKUef?dx^jF!XSpH~`H_s!g=LUhjHnKgZnn~R-brXi8Ka~oV^h&|S6 zZxXUEadNVxdq2~5{&$<=U9raycYkdDCLopNrwvV19IAVpy!)Zz$0dA?EK{)NHD-NP zEMyhM)CRFRTxj3nvbe!KcY|27*7utLHD+$L$Q@Uol%*^DsAH(p)sczL*Ca=j#y4%5 zYv&t}pNk$1^0ebo>jqxdQ4$e^RM^{Jhn4nVn~}L)+yLO*q>|p#Kw9hbsC;+*Ui_i- zFgA0w9=lqn)zJcZh~jy+x>~kI%qpx0|J-H|p33t` z#T@*|C7EbOUjxij6y}YdP3h;u%F9L>4LD{oGB70X3uTW{;@n!1O0=1v!4E1l^y04R zN3A$dmTZ*#2pWnXz=;1b&hfth;|TfR>v(Y2=6cLbFHhY@RNpzX?{M|&J?SB8)7%`@ z#@gk&ljmHQ%us`Lb8Bhgt&?S0C5Dd$Mj1Dm>ABaeZ4d0TyZuQ)@V;X&3FKH5UJ5rH z>ojz3C;K6l=xfol@~YOSqXhOBUee%5-Po^zl}1&UL%`u#1mn`>ws7OOL)$(1DcDP! zQs1^o+tV&Hkt-{QvMuW2ByWhVxBsHm<5kPqdAe?-% z?P<^4IRgwgkYxJ#g)=k2z9lwQV3axP&j=fpxn0RV{lk_QU-5my8dQ!En|?58X^rnC zf>1_>Cp&zK3G*uS5DDSF8yco$X!^6}R?=!P(iC?s>s>UCffQkyVeaw~C65X#XeBeH z{%M8wxt05@f2SV@Aus%=8`qM3ZliQfGr9tQ;I7MA!a~LJ9xl6YSD}RY=QfKOtRIyW zrr$bwP+*jnAGu|J$GCO)?P3-?`tmH zp|GS0(LOel?tx{HyhFHTP@ua-f&r|qmt{yGkIQwlx!qU1NYkE}mFdDO&K;baFV}ut z=ZSEvBkw=kE|G>CdqC(Yc_D1roo}XTI_k>GPYjw1Wi} z>mImM-&lYZ>gfXAYU#wg`;h5_6p-^y4dA7Th*}aX!+&lPYf`eV!%FI^hb2VclON(*%Z>bis|q|CPmgdIa(Hx!-#ZzsU#_Y1iybx zjNqW%w*%Nv-~}toWN5oM7;ZTIcW%sHae2gpmZ&9&D~y>-e5d3o$~cy zf|z$K(DPj!$$|c<#3Z2bYK0xNDp$5BS$CZRBPF zBpVqQQ1c6>vz^soB1tc+mL*(4ob#M8h6{gSorp#);NoIbbtj!iwQ&l@Y8{d3A7o8t zp?ne>7AN&I=L@Py>G?>@(i)S5Ket8uimLJiTxi0?R8unMxRsE{jgk)8fh(q`Ag9%4 zJ&hAK7(H3sKyaqWgBodBa_Y6(&+g?&QxkbwUQqjP_vPotql6=L$%!WM@L0?FjSedh z;%;|Ss&iCMKVn#p4lkk%M$E2$;O0;#wil&aksez6IE1B{mfnF2qf1%kn{u0@WJ`K=aeqOr>|gi-DvDl6dJtMrV+b_i+5dpWwd8 zLRPQBi`-7eh7|js2+0^!4GrL1U65QN16ox22`%H2#BNAKRdivTq^Tp1MwCOamDzB} zGuUwvieT^nG-juX@roD)U1hK`!|`MzB#@dsOqCPCvI0^CjV5MiJ{ zg2G=}dphTcJS&baM+!QQCuUgvLt;wnh)=FfDlUMW%Cd_D#aBwKx$7{o2((q-)H&(P zd}1ES!aI3kV&&$|yX;TO!|@&Y7WT*7P2ePdFHtnC0JkzZY;J)xb~S+=MTZ6Qj{VP? z#c@DAmwvb}1L*DV?Msca{BC=G8hI15<-mEhsS$j-%dCl9yz3gH4Go(%YY_65Cr?^; zd5<7MWBA{7*WSS<<8&AgsO|po2B{ zqlOv*`X0IT@GvsoYd2Fs)Pi!j=MIIHJ zvWj%+2>2zN;7!p?4r=6kV6C5*R29;6pNYo{o%Yj6lUCv0-s4t! zwwyI{w47{#owt{Qao5z5R9M>MNL#|}-azP&1cP6P^jdD1r_O~UsV%YDeMA05UA?Sz z0!Cr{Vt>S@^BX$};NN!Xk>+M&lW_%!h}=Twqo^UlnmORAJhv8TeSQ|!)f4yG2Gila zzt`XD;YdTANs7NSg#?Wf65}eeV{VjguSQvveII!Eo!79=t2Mk2K){O!4oC4Gx?8vg5xX=rzl9cCu45f~J+No>Kb6u`=mGplLR=+ zyt5;2CvKEE`6t;!#+}UG1O>_>pxtbebo1_$n!Z^k+~l|`Ewt4d9(AG@HJMoin#w40fOVva3TH7lB|mFvE<9Me=I7X$tPN*?o8XVPh^ z-TtmsoR;R4XJwaNzN5L;LzBeJF;7qt-Q{dRA`5RJ;Sjwm(a6F|$*cT7TS$%ik>B;&uA$Sy2CMZj;E1 zc$VjEt!P4xG?5iofU;wfj`J>d)VI(iOf)vxKEr1jve-zNV#FUBq;uB}ph#iB2!n>L zo4lFYv3&9I1l8v=gZszobMc5==nb-BfM>Ue;twQUan{oJr?p-Y<6b9Sg6-pQt#V>_ z_BQq%{9~m5_*uFc-W{0JS+6K3Y=?ensFZv!cH`gwi8}pd`pQsuZZGXeot-0U(IB_Y z`>NkptLUX@(B|zBWS;E$O?+OhCo7Fmo~h(?BP>zQ)jU60*taJje(!h0uadON{PH`}|-Ae%Y^2Dn5m4BS{ z(#YWZj@sh!!c$$1ep65xjRimXD9yWkO!*8oWUXjifnx{g(^1dGysp#)9wfJ+-kRR} zRn8?v=rZKnzJu?I4KgVWj?6JYj0W^}2#n8xxr2wrS3VXXJ86T~vvcrC{~7X3oRJ@| zcNYIHYHz=W8$~gOX^;cOIs8^%nw5zx?eXH4{oNWivqpX$wji$8dD_c6?slxHKWd~w z1XKqG>G5PA+C^~GRikW&?SSqVz5XnC_!HvKtz?gPga?UDZpXiv2=<-L4bHi-qM9*B zZ~#HlSLU>0?rBT)dCL%K2YB*T4y>_g7W~w`85Nj|VG06McTj8?u2O)Y4+WY4=ZfLewkqmEtMQl^F)^O=34u!$ZKa{>+aXiBCluHX>~|n$fGa*x>vG zAul=c*d)fq>K}8ChBMElkm;`rKSdoiTjGg-g>v7>@=i=#z5T-9nJdSU&FalfoV-9i zU3g{7D}xkiO||7OffEEW!fTT5TxzGbKMaPuTh2p=tYx&&&}ZHtS|X)jKbLp*4gRV; z$#1;X6H(1;2_OwfR{1{6A%;Y*rmAe`q#{G41D`W6{FPy{yXdZENruY zwqd{z?osQ&Z(`DcCdNx)%Qx##ghx>{pL%^KK;$5RA=$CVng0ED3Ec zGE9PB8h8B0IkN={)p%c>t`YxibvBovY>jwBL$>p_bj6e9k>~@Xai_kvjN@93{nSeS zwf}RwrxtMiWBg)o^b#UWn{YWbddJWv`XFe9t|vhdI{~<$WUX^FGNuo4=p}0`dDQ0m z%Ryc|D7Gc4jz{%K$hU0GiJD8SqEgNF$#{5&&cMM#D<@`?-Ko zfIyvCU0VmX;CbSG|A32&Xk&QC_IS%61OcvHZ8;lI5^cJ@7NA!eG>e~lerZ&$ScxZ9dd zN&zj@n{%D8DvXk1cFJx=UGl18T-zjX1?b?Dw8>}l+zmyR(1dvqy);|Kk#<2OT_#N? z5d`Dt+}~2?IpkQ(bML5twp;Smq0XQ$iv*z!Rnz+UKjjPAHJl@@joByI2fix4%8i+~ z$&W@_($l=@X=wjE4&U`jrOfAiR@c_WQCm3ulK{WPeP>#AQkbpvcw@|vmrUl3hhB3Fb!V;vD1WwbMRA|B6m z6?HXEO$`ZT^O19DepQXD-cXP)mBLbPs941Kikd$xl0FqXvR!&^N0sg`3lKldeq3*H z)531ySkJp33CLreJaaelD{r-^7=khq)2}GvtG25D(7n(se)ncwfnzqOrt=#PM@&HH zRkJZI@HPx3XvxLr#EsIA#JJ9_oZF(=@!4ULZ%xYx+nf;r8ef=J&o*fET)yg%SD|N< zy%aYu#}BJ5ph43zjBDwu&iO5m(?5f}2p$T_XJm(l$uZo9)l8gX@?g$>Sy^PQOhUCs z+jzRzq?~N1q&%gK&a~$bM$D{MD^A(5yj6k~^}%sb2JPdbE+;E70SE@g8e{i)*wUYo zvDKv9kTCDu0vvJ$$^hPoG2$fy7k-gm_0pTw9=>l{bp2D>NlT1u=9Z3^>oT z$va<9w?z%P#NW85*x%*c%FC#??FfTojk|A}Ee7e1#C@?r*9wAm)xGWyp@gi4Z`Is*oV)&^C?Dni#@V^^_x1<*B#Lrs*iKr4g zc`H#t;$xcM^+up~$(A4zZ6S%NEMi-gR}*s&ztQoy=Lhj*s#tP7BcihK(uO z*KL}^ILY%80_HOok2s9bkq~S@NmoywJm42bDkdl2Fn6t|cS-RRU#chws)}p@$TDiQn<3 zvnt-JKbECD!Gtgcc-ZG~R_UQ}<7a-3Wg7_zieMPGVLlWjAAlGeh|HlQ8Ty1HUNB9l zjO;y8kMe^HYNM6Gb#Czn=pSAZ!VO~T>aQy~Xc%rDxd|84eLZ_G*2p8u=pb4%0vL0` z4kUT-rT4y^^yw)?AEx0T6NTS5XY1!UctCBj-hLrS0WQ{xU9LEb9tWd4$fu-B3^QVZ zy<}k-KG3~lsnG@w+`QVVBXxQgFj)-lDoCO!${(3)J)hqwFRfHK$uiE3P|pYu&fR+5 z{+gv7e2&oY^bhj~4&R=REI_s921D|JnaUhQG4b9|Tl_9#9SeUhtXB^Q=>q4#wD?a& zmg}zx@xUcqJ>2X$+qY>OILMPuGGO)>e|jD*Q$WcQw`{6=y+5ANQ2rz!=xkH!oSzAp z9qB~Mq;h|X+RKB_Q+3nVIq@(XpGhO}k6PQ7f)1(b;*9qC`FR}9;d4^@H#9owl$=3Q z$rqD}(&8!woeNP5&B{KlK#u|okU8A;WVZ+Czr9vokY(*A&%mDn?;EV0X1mwjnig2& zx$6TA%Zbird$tLi6FgPd*dWBWk$vzZ-xG5mO5r>6Y{@w%Ss6_J0whFk6z)UdEl zNu}EoA5!8*@EH6Qr2rksB*8}_fFEeOLhTq0voYLSiPs0n+7G(+B?CMkcCEOKv%T7g5ocvq$#s z`(az5{+W&mf(7Ye$dByQs@1yw*g3?SBT0Q2b~NeH?$XTeHK>B=Dpjv}-^d+AjQHYs z|NQgffI%-;`+f(&P>drnB3py<|2O|T0lGf_%fS?Wiis52Lh~SHKeQhaefkD;cpw&Y z|Dp}?FO>Djm;dL~lfNUP8#qNb`c-rCh@L*B(q8!)q^jc9R|=|9%aN5K=OcP)5-lwA8RyZZA3MEPR-}sQj8$0otuuHf?e8@a8doj@yV&!hFdD z(P~ z{3{SSBO!W?TiSLUn)yEW@&(DR8%beE+&kZSm0*y*(RU4u1!4X%OqOEXK)^?Cmg<(y zT+UnUPQnCYeHdeitJ&I|`Dm_I;mF`1UfrCi@Lt&$v6b43-(6{nZ7I2Xv?XTR#%#=$ zAT5B3%qA8=ZYnu4YWvdm_jQcSO;lnl!EAO#Y0Rl`%k_i6GLcIGfg)1Os_EAHjN>A} z9)g#BW`7^$Q0Q~PrJ8-tAPX69k_ig10IF4VQ(o4|6ZzDST)y5ot7Ev2WvqH=!`!}X z8W?m6>O0$_HVrNIqf6NHsLQ}ly`;D1(83R!7VN^t>!aj>=5t5#Tb++W%XMy<6m?CfDeO0LY|5OxtBX7&J4h$ck}oh2+d9gQTRwq3(K#Tdo=|01MXsZE-NHnctY1+WQYNdGl9pWl2NyUlY?62zw{;%mHy<|BWm&-EG z&rZapI$zGS<>&|b_kAevH@CrqGPliR_&zdwb=qk_f!O}^&utPLOeax{RPDTJq}I3! zSPAgzitK-!ER%VwHGwUK7a&QzxaRzUf`C`*1H1Xs43Koc#>h$*YAb)=9j?~K+`+uR zJPiEiq2P#s`vA`L34pBE0G7ksRwMP{_NcGtKQ8`rTN>;wZ|n1QXQd9PQhwlE?4-Y- zuYdC%BeMjDyx&lgdGgl(TZ6N0ejL$`&VX+QI`ne9NL)&POImOsZ-o%}uYTYEclLU% zA7no2ou8SzDBRWk77qrzGOJ)kYPtMVqOjq+L4=ynCI|Gw3%!U zeE?-E@yaS?lC}@8bFk-@?;k08mmJfK07d0i)uO!U1>oNzosOW+OOB+Wzhy5T<3+KW zlJX`M`+M2>ewMgEMD_$>vLB10bA_QWT(qbF@1pEBTvMC~l-*XmW&PrOA9tk)lF;rH z*1X>>js?nfxMkt7CVccNicY1_JAw*e`bo$hLlYTPVh)*XIqkHn;1e1;jBbkn((?;g zlp{S4w-^`ZT3y8YPFc$SHUwAlqLrY5r^B8Z@FHkfCU9N(0W)xlMz6)!lYHj#r zXcI2O#)k{OIn-R(*Arv~k^s*qQJRM*)UT-^1%N~HWzc?(mNbFjWAU8=enwK1q4Yt< z+W~#C5~>}cRp9Bgda_VQy>yJs2zZqx>FcDK%I+K8jYFKZ4C1#WlX8TXPJ#mjofGrf zI@l`6pWFP}U4TFlTOJ+&hM99efo6kyj@gZL(tgwb7Nn6SsjFRK?8nlF23dOb*9Ha} z>vQEya^gc3cj8<(YzN9+V%=?M(=I;J$q}U`Y-M!$wS0kobFia~ZLPC9?R7+I!MkY- zpllq9YNE36tHIRB8SCtz{okmWY(YV6wQ#s4#_o9pa-E~gWQ=)|+#)w)B zk)+=-x>q?23m7X3$VK&xPA@vQ025&fgWFC!kpC8QpH%jD<@fEQD04xUK7*;{IXdYz z-qK<3ku5kcs>EGO*i1q7X4@ip_TttF1&JNS({#M}SUt8c%74c%BBG^;>W8d@c{MC- z+Lzqm)JHYJ{CUyPtYmGUIWlnBmDaX=r`Wf;Io-*ERA99<0ms=IS=S*ADe7VNH^cdhgKwNb%zEQpsDxNse zoxkwVYo_MzGAfo&|BM41Gonf+=6Z6J9t~d=Ukgc}_DF~3;NmbA*XW5NACC&zT9=dY zxKme^EbE5}B(>m~!nR;N)_s=D&5vpo(t#MM1k(uyQPPVO4E^#gu~T2Plb2-0yVNW> za<3|#8{A&pBClN2Isc|4G|vkcYdtUuHkkbot>v+|2QT-iKa=e@N=F-UlTKY4x!OK% zYi(fjZM{8Y4GQ4LH_VgsZLlWS1{eyuwW{Rg&jx*5mfC(8#Qq)!Px+3KPG9zfzzI&H zt2!kI>0O6bR6L~y;%Wr_3+l;uKGq1rA|Z}rKnf1&9M1%8dYA@=)i~#p%%cUxn!|h{>S1!DbW?5Cgsb zDp{=J`7^W#gGvc8as+6C=qJH=Ta`Dnc$OLT z))_+zuUF+Lbp^=Jl5oD+Yn`zRz!BV~xtTOh;_3`}P54K=p^x|xpos1mqOvs<5*B5; z$El_w)BM=&f9KHlH-feDb^|E#BUPW&O*{tPuHHNa0nqkGKP~}r-2Y zpK|->FMpd0|NYTl6Y}3P@!!wH|F2$!^Iq(}!pv!C>U=0?l~wSUiQ0dSE|2Y7=YxN4 z3pbTbNPPG^vaD^-)x1gCxApE4{QL~pbi7Acm6nLviHFX$zpHsBUCjR8&j)TbSEI|Q zHRx?CUr?KlAX`%XUzrQ7wx&Ix_{hn)SXTzAfIjU%huSXodf~jwRs$#6_5NsNQPI)* z;{ozfPh(dCC3@jNYlCQI5-gnIJ?hDKZks5W5FH=TeWTPf8PtYO(k1yxbu*otO-tCM z_B#RcJOn%~G_<&qJGTPF_ghtS+@LJ~vNsY8Nj{<~(^bG88oU??9r@Rf%zr6ziNgGE zCx{E!)oy-S;bgr@cX0lo-49=@c>C|arTei}=D2z&OGSK@vn->tmG7S&_%U;FfYAn> z3$1h5eWzLmb88-r1I+B>9?d!Zr@jBIsbC-nn8o=K^;HjpQHKb(6au|uI;a_$z}4d zt!aY()J4ToE|8bVp3gq6$pbpFx)!t7{zFB}h+lyx$v?NN{|rp}|FQr5SCRMwhVD5T zCqX;yH(MRv-CHLd+FVu(rngp4gcQwlR`K?ZtOk4?^!-f)t33HgJ{-Q^(+1~(aM3Sr z&6E?*j?Jqcq4dk`$dV)7Ital8l}Z|0=jhwsKJkxdumXRNuuzVIPUV!`a(Y}rOCELl z(vxpwkfd!Q*~AMxHG1qF(jU$(>3oZ~?AB~V(GP=Ka8f6Oy=O&BSy+ep=v&DQK z*&KD5`Gi^)PV#jmm7IREc04%`@d0D%eM@@NeEuWF#ih{&LnAWc_>WH|yC0|4oiq11 z$ebWl+2wzsUN##&KL}#G0`wXt9v*&PHZw!i4@&a-X_$KY@j^*8{&5g5pt>d67@BOk zvd^GS=o2x^EtsmiKEUfDWqs?EUO|f#_w~>Z$P|6fTGsWsH3$%eaEdOzJmHI$VU|W%Y`?EY7Y*oZey4}?1@#E$w z;4MD#52u2PW*a||bzI7UcysT@ag#JX!9?iV-!CEb3!t*6ydMAfFF&xr{?i|yy%Mj^ z;Uw}4=}{UbmS21gr#Q>=ZN)L{Zn(AG#g^(hQ3_}ff2K@RvI8P35-*PVu`D2L(W^Xe z^i@}1+fQc@`1qaV!Nz`RQT(=>p8v61Z^XyH{)d5P-M@QtLu9yaJDRq5rKWzmm2=u@ z+M_r}VMf6wi_+C6T^dlPk{u89T?#wbzK7mPuIzUnBtTo1S4-S7T$x+5-wXs~}8+Y}D1-gmRWtPeNr~4N7O6SIQ}Z-Ch&>Y6K%L zq4!mJf_T6LCb0QvVce%sWGDH2uo;7#m<-OU2!_5PS4ak|<-NqO>mw>lH#F`uegLEFcH)xCZn+KRPa zwyjUVh%bHB=QCb;37R-qj*Ax3t-u(Hq--;rz!1-tkDv!A9)eRtnt)ot~Ix~;mSE%eBknYG&&HBSgZW& zhJ8qE(BH^1G|e%;*I!R|BBv-oJ)EN^(N7?G_@D(lfJIEtJMl)XQnipF>o%d16{DW>k zNm#{(Pl6wxk9&9~q0h8u{JL_mj^l@V%kf1`to)m^Y|vFc9h(qjB71_DowhK!eahMH zRI2-|iaRwd$3$R#jBYv~#fSV`~CAk-@(0^l`-cOH05jO&$G_<1^3ZF4=dM zX&YT+*%FMqOU{V1@sFXV%Kq3ygJBXrTD3X|d_4*foc0SlDn|mUizLpS4zIAaQJYT2 zV-4#JE!kFkCTQ*axoui4w9axm^xjbODWz?k^u-}NmS=79_P$~hvjIv$igFOY5&CF% zUFO?39@c3$T?{D}B$JjbojqvYdpo$D<`$Gh8_P9JJxt&~Izc;<nZDsfG`{sXpcS-F5L_U}Sa5VAV4WT(no2B+IIQ?~s6x2^UL$ zXv+gj%Wc;y9jhE9`!K-aJm4f+iP}M@FxJlcE+)8*8pO1@)Pjmk87SB5#~CRP zfDS;9&l~eQbCYuqY)8^! z*9#H<;5qL}>p0oQ*Z&w5&f?-M@Dv#uRJDSUe26?sbr zWLlv;>Pc1!2~ch2R6Wg1<}q`a%_iPeUaVr0=q!IO`z2k<0M7;bIjlt`6@DYm^doIP zkTrkz@LK#ghXV0&2NK<4$g)J#K$l_!2$NKPGdd^DdeD~hB%&g-`%y)kKGMK)s2c}5 zov9Bh*$Yn>o#8ww`Q?a^le3B5&~!|P^g-i$$*~v6j~Fcv?+A2B z&aRX+OnS;jg2n0?uQ)g(wB)o0Q^oro;_AC8T>KuPyyB~jY8|K91}dHI{FJQ6nyAjd zxKCN(;vJR~J^d^flX>g3L*^RK0QJMOVMq61{=*$-aNRVMwqJ-&FUg-F4RjVcsD2aR z4TQq4lI*+6bZrz5&9;#HIQDmtEsCDTcJjnwm}O`v*RcPYpLS6})VB2>%pXu?ZEriA zct1UBm$()ncURFuah`E?%?(0Ik?J9@2cqZ(?rVa;P(ZMJmIp_l`Jv@CkaJ{?ic2-x0Qi!Q-yOY3hAU zirU}KJK^2DKPgyp`G;olay$KA{J|sV$WTs}ZNB16$b;W!LjgH$UXBh`8(i3t=yHA1 zXn)+MME6dd%BHXGjqJBT8wV_@v*rBLzWD?NhbA&gXaPtc{Vqd`@rD+aK5UHSv2?LL z!qTeH)n!BO^~1J>_a7|i8mt8wIn}ac0LkDiVAxXok~CVGW=+}bN?sTn5OCFI1LC2c z91>iwH&=i)SMO1I{Mh2@>e5clh9{7KfJ@<3lG|Xt{)tCg?iFpR#sasrc$MKAWDMm} z6BNmUN!DTyj>q2BKKREFbQLeEfH1V^{(fJi$;t6(F#XaI=jweY&WJLs)_ zpI@dMK|4C8<^f1?JE>ZTwXI?tN?$+aQZEe%#YC@}SJP!> zd`oNh#tAE>TlI??p7CkY*c7n{cpAOt^?Y|QPr}Of!Q6%z321GTSa+7{QCmn8 zsB_f|QlA((OA;_uk2v(_wxpDl$&^tIGAh z0t7V-g>N2d{aEodp1zl}V5l`tJ%eoBVzGNYX7|t;$LbMB26#fu%{J$cQLYmEOhlZ@ zYZL{KV3Ke(ZUsdo7`eGM)uVIj zGb`<|Fl0L=c7k$Y$3>TzZ&9vKo7yIQ0;pp)xIicMsn6RGXP|>&SU=Pd07l7a!cbY! z%;atA$+t6tD=b93=PBVu_BUhm?gy&E0*KRUZ8=Vr`<-EdON`W68)bCW?xAL1rcJHM zIN;6L77Deiv%Q`%hPctwq}PDTsBM64vO~H6(GB9!jG1E|;xjbkKevT6^;(pyDeQ$t z_L^kS2?JQ6hAcl>0Y68x|LSfMF_P`cT!x!`%jH^Jdk)k`l+MBnJN#T?L5nSikJ-Ux zvzO3cjm6=!3b?xx6Fu}DKKM#GC#ve^P=ON8d9l-D!S5t^vN36RSr|=yUv)|9>0I1; zVxm>aqd50`XISZy98lI`lxC@PL59XYdCiX0^}lknO0QcQ)#%1|>hT{CJ#KMQfy zYNpM1=FPnMl$+63LFf>(trf5j%{@BH3?20aJdL0w30#MrP!@chCDlrGi~*`)1&JtL z5BUZt$=^C{n7QV)PkB=eOSeJjdKdYUI>tyT6HT`Tqe)!ddHF+JqJ|-#q99EWv2nnW zNuR{B=xHdyo6eR-3CWkg$n~RBR+~uM;y(8=uLGfU^_^OFBtgp2IR93DBjL<-fIr_vq?)RvgZmx1DQglBb_wIUfz-l9ED)RBM+}4l>E( zq(1)av^8s=^=GfUVoj^=*9KZmfaoitPRRos)02^aA*F@2CZ_W^*w~CV=Q1`wiBVN< zrmdnkr=mBwtHOHcoyva3Rs z`s??2zBx9M|Ri(}cARU%Rjpn`%5ippfkR3Z?ELO?`BB}fobii{B=$`BwBhXND?3J?$|L1mN> zB0~}&kSL=VkeNtAh|FU`!Vm%(yvshP`&Yy5u0DOb&%M9<_(vX}CvNu6Ucs}QS z?OoyaX7VhsfY%(?oneIv;)OB8)}i<<)8X5Mp@TNIqD?#X>QZs}EN^;3-Hy~vZMRfs zp$d9m^?uhEbln{}xMkIXj{?7u%v&02tDB8ek?-<{Y;}3fdqBV%RU|cpYaue2FJ89v z3OE`oCpFPDL9lw7lXocxUo!C*KcUr$Lcn)YFKdJdzN`j1k=k$tDGf8{Tn(BK316`` z2z&AORSIP!s{mC`2Cv*s*^XLz9pY~cy)(pF<)7iW&oGN&rkaR&Je+tc31LY(ja#A! zKxT60(YduLp70GOHjsx4K0);Zdphw+=5Ze!8D2`(Fx@oc!ZOQN{S3-ElruG?=4|x- z)#(JV19ck?vuN-xv%<1rC$G*m?;Ejnb$7c6A6!mL!_vgJcsq`J)y#FeY)W|% zTriUbtLoCxhD}SZGWwmc`hI0$6McY4Y``r)w>AnTFF=-bidR4LN+;C!Dvf0X1qZJI z^;z6pBEq)ZvQ_=t=B0GgBJ6w^hm5V_jn_VruY<)UVBASQK{@Yj@|ZL~|9L$Ntr+!) zK8m}$8G;tT3`Ny1V@8(}s|MX%kL0zjPjq5CGzgMF8r8IHKBrqC_nm;IK# zTd4Zik4Uu%3pEN3tDD2O3gH(!!gr%m1aq}c2&Y&z3sN*xgjctF3yQwh-lVS?#zaZA z2z-kevD={Jpd*20P;rb;o|jCV&v@6*MgJkNN=C0b|gTRLWy$w}92VbrlMpT3a_ zUh)$>gJDy`a?$Qix@%n2!n7|`WA+*9g$y>&n?-nyM2w=a~I9A^;crQQ3TVEKAW z2R5u16jT*b>*OO>dMGbGOxs+r;%gbXC=)6kKRJ&^?4^Dre2XD!AIqEF~+UqdZ$cV_YETg5Jt37(!jg|cA{@)u(AMgE+a(D@&D?b>ltG{z0+R4P>)@G}~dgb%7! zu|h0V+TMlM3wqwPmuZn_BtA{&sJKr!R#OMz%bq~aw>bV>m&q;p`Y=o*WtUK|P{@Ga z5C5`jp<-sxI#Aj`i94b=);Rnw0i6-7p5U0{Oq&BW5|xKNn+q$7!b8y-LbqY$-6i-} z)wX!CX0M5lmwa8>;L%v+vmkR(b5O)dw5hTjOR#rMH=JYGuyxq!HS>Z)ytVgDR*Cjc znkn(Wyer+&`ymO+%1MFFeV+=1kg?Ktz0TB`Axix7n1|w4Ug}ITe-r(f&~fl_xn_*K z9BTYzmp}D>FZV6ZL-OR-xTck~yFGI;gtcS-1|+veMxZls&@ii`Be_ z96v&LFR6%z2t}@c2u!e+l7X^^6fVFV+n@b7?!r97_HL$el-0eV{X)NBooK3pawCR^ zOKlc;gEAZe=4dl(RDx}hEG3bJltnib{)0ok>I1MxRtk`4zEUFHzlWN*d{(XX*&*{@7z%-!7{@G__;CY~pu8zQa=qouQ_+YL0ynnvBa zKAe%1fnG?B{+)18op{FpVP`3y;?KSpZp^cd0txT$yV;j|$`PaBIN>?7(N~#cBTc7( zA3b2$ixiFJ^4p?JoREX;2bl^byv*zuPcWCIz2)7wbV%7YM_gYr#b@B^vKBdz-mPzc zS^j5n@CVzC)3DSA*1jgo3XOVA*U&4W$xdFqUjZzlkK_wQYNv9SaPjLWl(J&O)%&OU zsGtkHeaWg(85SN%G!i+*J;Cwx!4&6`LU9qMrn9CKOx-B4jnJmhb}&}fy0q8iIG&73 zubm1dXXGMczO$zUs|DCINwW?+i_yw_IUKbe$6zOY<-Gho`#&))5|W(N1T!i~Qh>&NZLIL(EplPIcUZE~lH1&?7>ZauNU zF{4A%zwc4np83uyCpH^8*JJNUac5wGehS?DleGd-R4d)vWw!sUzWj!%1Se-1ky!g*>Isbh{QdCDMmlO9rx=r-99%)Q_SgHtI3kC_cjnt`H~}S?l3`!J-ZRjS-Ya@rN;xEmsr&Uu4R(qDc zX?t+t_oN;05_TP);jJ8uw!h-%g=3yN&vxoeyF#{{m*QeHwPI$bm^Yb$T!Z;<0dvv_ zrIo_u%R1YMkCaD}!;0LLEo(c?0=uZ*ZGURwA*~{}@998W1IbFh#cxz@m(7IvC9Jom?HQB3~SI;D}m9 z@cAr-Uvjq-sKNcBQp*7VGFiv z;_u^eoEdZvbIjs;nQsE7kEN$8DFb+)hIkWX5E*4qfFwwQT=*BiA&z#9cyfOD2@`0m z9Rw;!s%Y@*6D!*BQT{PDn$~4ewp~R-m$YX%!yB8{dmkz+qy`|2lU-@FP1!1WPqx@y zZp@FFUJAkAqkK1Iyn?d5OkW_kemT(U)_En)rc|p_PZKDi^GbU(=mzof;>+D8XV4pm zY);fc+h?xEX)2>&mg>-i^dmu*g}_&VTjo&JarOgxaJX%iFs9LL553y@N;Zoz_06R5 z*%vvo5%vj4?UBS&cV05ltwkLqo_Nt?FsLpp=p>&WVbFTI_F^^VB@i*ViW!XNj(M_N z0f*4uLE1-e@4ir>Vm^FyClU}L6*yv_{0Rpa^-iuoeg?}l)wd9k10B;XN9+TgryBYk zgD`|k;fK1!&U{_-j@(`oofJdaA(q+DPHkfIjgcr_WxAC!M2QGdIJK*_Z^;p&e zjQrwzv)#htZNyZ)q<#3!*8#$t8j0 z8xR@ousEc8NNgnZ8?rI9m5Dw1Jvr{EG6&sn5xj>Rcl1Z(em#phU?C6d0y=}UnQ5O+GQWL-F)_;$%XKrPO7gg#|ci| zxpw|dLTM4^AgI^{Tl%VKnPu;~S1@Ts)~2!U1^G?h0@%BZ{HAqN=V74uU;nTnhXPyn zE0v5w;gWT@lY_RKQTJ+9bmJX)0@4ewxyxR2ms-^gL&8bAkCpql*q<15J~+$+%2(Z^ zyr78k8Z&E22txH6uch<-&TmT3UUjnK&>HvXpG?{s!WFZi!mbaoj^!P<^1TGc zcWF1@=I%}qXX#4+^xk{EZH-v3#|LfxVK24a(VwFWDv?)HdpUiN_=F==hBbh=6GVKA|iAqR`s(m(+XK~>h9{S9Nd&JGo+ z?Qxs{H(?zI@oY4%C)=&UQQ6;*3r)I-ZFsVM0M| z;lqPqNlrvTYFApbW|~y)1i0zL-cD?0gvEZ8=?%`j91U`h69!FUEnX`zDci}b10~uNvs^wh1Q(gcJAX9nK)I+@r*q8o zv}d+G0-D&6Ou{fvfxHVgjW>I*`dB&~&?h%gZ{y2Fcv(b(b%FQOlOy_wu`d%1p{!j= z2V3U^u$5x)Bbx@cbs9hv#HL6&uT1&Mt*K^OiBibM=S>1s%27_Mo)!7#8?3Sg3CSvs zYRLSOFYA@{4SHARH&s0bs+*=`3oZ#G(9}}%^$&}1>Y%4Q?HvO-A7uc^vxA`nMBGKk zJ@OqJ+`PuDj8fNeoLd?mgS0p}pq>VM3(S2pWkg6^Sz%7qMLIE6#kdn6V~o}6Eveho ztcEF@BTixJAyFZ$5vro%jcauX&$7sj9G zt7I3UkhP2j<2ZtsyGaZ-)EZ~Z(-oIhMK0pyL2&3134CKHO6r?6f@1ZVTBaUY60y0*s<{qk+JyPYh{%CfmAWIqJL0ZueL^Q2{E(mI#H=&ZnN4;m`!&Ojs zb+-%ihMeR#dc!>KFVLJkUE8zE5^PiDQ}@=?UDnjsBm;MK-B!sczTlHhmA3c<&m%@k zEATW7IaQ{egmP!wlx0{EN)oM`756!+QK}Mxe4xy^CLr=c8p8&Clvyvf#th<>`(h8m z32@X=t1{AjApVKpmYcx|TFos^m+%WQe}A;y!PuKwqCpw zWMRa}VJYhZUC3pi3(F9iK0{=x2xXspr7ErpU^*KNJAtZ=WOQ4`^_!ToB}Dh91%&!1 zl-+rhvVC%}S-WAE4`oW2_}d`@o)1-~zzVxbw7qkk=}{ z;5D!>eQHc#YSzoOEo;{AwD5B;)%&D_`;JAcqWt{4@BlDfWQ>o%U&Os!O%Gchb7@$%c0W%mi{5o;U;m{;S?q%&0oE-4nWF|)a?O8Gld-$( z(I5e6ZyjbG`U-Z^W^^i0$)=~ZC(Sl;9hmY&93Y&P5*~-<{G%Tr3g!5E^?#dh!G43{cD`N`pkc zFz1UvDCd<_Ul{z_rhco6a#zO1?tty~<~e?ajV(sL+LciAe#7>)1$Hi0gH*=mvp~pi zlPq2i6;-zqUZatkGnI}j@?)}yoo}gH4?D^ZmgzpmRu!Ud36eVlniqoId2RRoTU9T0 zE1xOpEgt%j(F_zM5X!+f^>ii`l7qB&JPkEas1qL(rn&K?VJb}`BRhHk&c$j^*Fw!< zeXqPZx_@ldDSLbS#xeKVvpjk|OU;GX%@}304Q3z^kwo%67dx-LMQ_91!dgkf)G?@= zPxmUO-KDMJhuig;KxdzYv?Y{P#^yy1g;w+)QBw{&Typ)e7=-t*b&rJ^mT`?vV?Ec+ zA6*FEbk_F*EzX8%sOS622LK?m=e}%JNZ9(o|H0w|eH6jzKmdw<>%DfZK2Xr{oF^-s zhwKpEwA9JB!JI+)*k8-Eua3Wdu$ADL5vbt!Xz}TW=;-XcslfD_pB+-Ro*Q7CCTC|d z%oB~kdY+1N0U-4Lzy0}dd({&Fwu&N6qtopRE`O+-G z=Y`{BO5hU7k3C?qE{7ftPaU`1I+o-u27V7dgiwj5Gz~M{;~jnM-`m#zj{f_6&;L_1 z=X1;d&$j$u=%W7;3*DP=1(0OBOHo4Q*DL;1XV9CPciF{!2Yae7J2vekrmP|OR1)u~ zeNV(|pIfIRo&`X?Ocbt`v{owGbddPEpzd)g8OvJSX~wiKtthL$qvNBa=9zofzQ0sW zuBfQyb!+gPv^8tPMnimLkAvv>+%Rw~xUf2+w_2`9yer9)zaxb5;KjkIUc>WF8j-l` ztn7-4I2I8otrlVT%?jnhHoDx(MX%;5#umc76*G6nb5atu+N})gyMoW_vfMVz7_D3I z-ll4ZMhZQEg8DdO}R5O=iiM*kX%I zsYLN!(@7Ok-j8O`nh-VDTxO6YWd08JSHxgHfE`%(pu8x6c4@%r1f z-xd=vg(eW!h>d2*(#Iq2328*ma$rQU(+BD+r>;TO^)^7n)XBWdz$x~$yxi&bCa?9h zAia}nKJhKc>Qa!wXkqh|R*baRmO$DV6d%ZaUS zn>}0uyJ5*_KYNdhxw0k`z>i&4DFevY*oi4{-(5o4!L|1n zW_M1ctPr0R6La%Xal_D5M^agdlVi4*I`Uh*{L1$+d)^dwNM?oWaNdq+r7$Cbe|yn5 zFXfp^b$aR4)~vomOu|rU|Cx?f^;>Atx+%(+ez!!<2l#tk= zqkOJ1aodNoxYn_VjL~#YSNBUJzCi~GA<0`V-u~pJ_qRUwPQTDH@$UI$%JTLRSs)_= z>gq;7MjJ?#-slAqGt$at;IhXcr@sKVRQO|V_8)P#|9|%X|B`^R#qLbdy4=ScyMf!P z=E043_~Wl#_V?&>AZL1V)5p*Gq#*Z3bl>PV?7gnl`b}(UVM91FgAii+_et=7qahWb zA7!M+m*9Wa10wudgZ$}+6HSs^82S;&f|J9o%Lef)W58bVCT0T2LKhwbG1L;&q8yWR zAq=vK5RM|mZT1c$EA}aoMD3w;tVzjm^LKQus7Rnp`z%u~^#UB}HoYQ_uzUrGL7Wj1 z0Cj4kw4E($wjFivq7vqwzvEy}wUx1&%_u;KljnLI(^9_bOZ5h6AIY_69$pT&v~9~85_ zW1WJQ+`Ty%PpX`?2ox~Cwpy6ImwB)oj}w1A*m}g5TE=mo6OS3j+~86IBD$Bvtqz0V&lPr#n(kaCg%z$Y)VU!ar> z0rE+Elg^8RzR42H0a$a8ac$ASy?}5#A&{;Q?ZCJL*JA1`yqpjwU9ny7Y>U8n zO@dH^`WSV;GW39xekgUo9hW69K2##90x$LyqF7|Mmp!%BWh?LKjN2lSd!ygHEW-qv0x_DmxUgZ7C#E@1eEFkHiM!~N;ijCo-Yd)xS;GA~ zSdjyYSs$**hsNYiz;d4?PAOpou2w`;ea1fMhSqrO+DM%mqB_m-oKlB|Cg7y;HzFpkjzXjUoF} zq|^FXvIq#Wa;F%QQpnX=x$s8n*VPC3kU?KUk*CXHkZ48(&ocOFc*h=UFiGfiQv&QC z9g`n*&qaN^g2iAExiG7Mpj!%0%B`!c#&6umO1q3sFCzq{l9!9BcYwUc5!XSWUQ{1~ zj59b=RID?amUF2`oe=M@t?U}KauO5DrPHVcqv%B>gDto#`8izCyDGWy!JzIIRH!X~ zU$&R06rN_#a?@(pGWCIL)b&u3kC#%VJX1{o<2B9D_@Hc_beFrht=gz)I!0Gu%3<2> z#MK8Wg2k4*oZ14750cz^{_W~T`h^~?1;i1Z+2W_AgoojChzE+_(IsQv`mN{N9LwXT zYPrhOUTbLvl}!uV zh-<;Z7qr2|vmonw2J@yzWR7{?ATNDbjqh09o9FZ>3ao`{|6{5=pOCbF)8~k83}nvQ z#Py@h4~RC&rXFA<2qwZ_W*tPZ_)#XO%z=@mh#5&iHA!F5vsU(Ehp2r>sWmAQ!vSEW zmHSiqcHpLS`>@#j*4~tRFZy3G-R|~UsUc6MiY9v~|ts2pM`zha=xcJsz^2dQhEYyTwJ z*bg$WClTrgOSLjia2=Delg9IHkuF>53t>(TAx#gIZEP$qS4R%dFfNx`t=HUHeFNXR z2kVQ}4PxO>KD#sjQKpfjQJ<<}N~}rL-Qt}o8*_%aH|rk3wse3o5FE8Mt5jN!CHb<` zh{?#53a7Ssn-8;!!m~C4yv%1od)Iv_9&KBme0m<;LvW;l1;FvK+U?f=E zXVN>@fKM9>3tG-HN?Rh?-7qN|a)=3t*=gGZWqT$zK}(ZrW3&Ys0e(tvx?B!}q>Lp^ zNnAwL9e^&A?BpBRTTn)p^t`)zFHY*lE;=0`_?*?u!1^2>Fk~qKrQDt%G*+zXQlbGC zC|jSfon2E~Z?Y}&n*8*a&U9L8U7OdY87+~*#-s;9{JTR@sf@Ej-oj2gZ1PTpQ5g%Y z6V|~gi1|XkN8McrbGOpkm&lbj$m!&UclfrpKOQ+)IS%1U7*o?( zJou0PZL3i3v=bdT2HmD?eaWd#{n+_$-QC^yrlqBUTu_bknm=z%K5b9_@bk+pE$&y_ z-G8i_Z}kntP#f5qlx(Px=Q!}yK<2TbY%qQ^b2{8V#4Jrd4MpUaN2|?4J?(=m*84S* zgTlt}1kb!Dw}o|r_mX>?hf^b~wIQ`kh#X3{fw3=#E{nP%T7Uj~+Pp6U<@hj0{YI&n zI#7gXs0zLnW(})GtH=pEc+3WvZUja9k{8cuir;cgB-c&bn$or-)5+Jlh|fucqyhaM zQDXFw$Euky_{_I_TD@vHsV?gwm*-!B2@D6bh_aVPw?ugk_cCeWM)JeD` zz-oD(WpZ11^n=EXDc=cf?Kpm08zDPT%u{@SQi>kN`*P#F#ZCJ8lcStC-My}JuE6xz z?Gw#_g)hW-%F9dvLhRS5@z+(~1J(1)%IyA0LFWKj>wtPfGBhFCkvK0t*fY81e1$5| z{i^$NdZkIc`J;@jP!&6+w40xoYC;znKibqHuC$cKD9V6sPdAjX`9=eILL-43 zU92Gb3Pt)bO38gr8>Vh*p$9QCT8-X6WP8S@E@i^g2v;I7qgewixSySc-s++FEz(z; z#HS<=tL1rFF1^>Uo45^rfB5~had0p$W!OLC5!c>5Ho0=p(b1mbLw0%xV&D5z!6V7j z3$@bCLMGp+ZeF$B8f2@U^8eVX83(p0h;`R_R_!-z66-U(dYe$3e1nSuC;rV#Jnw^!>9h2nJU4MWOmdnbR*fWu8Y0>Kt5+8q9E z8YXI#wS=YbU)^RBz`xn`TDqr%C&`N~6lT>MK{+V>8kO%DmH=*A?$m!+h$W8##dhzS z5143}YP`OuuFt-dMu)2OhN6JImwbmR<0aR$RWQB#$7M4$Z}VA^)DN@KjMSLxTd3e5n7 zqBOV2uv2u~^d8(WRFh~J;98lC8P69P91D{R9+Jq~lx5D%!@4*{AJTAR6`51|aYv#C zV$(84kpeFrvGX&pmrPnv(WKyIzs2}Tb-G<)v+7%;1KRVtEmR|sHT;mxAY4h=po+|M z?lvT4YW6Hz5hr`AVkPNsOC(vexKne|>s94hhAiO3*Df^TVW5%YgI9&vI;{@RD&LAS zi-NixF$n$Cx4j~umOkLZA&0}c9Y2`T8?qFuO$Rk45ZjbFP1fnz^Mp>u+fL5`M6iZW zNqGXd9}AlvhRau09w?qH(oV>!&RgCbr84VbR0`0Up8fsVbxHtjer$ zQC2{Zk;vczpm%IFIWnZW)+<~+eMNgy-8nC{I{{wVcha&Gvac+aJfXR)`hbj5(YiZr zo9MIod}L4fp4T{C<_{!NvJa+gz%VU4YRM0{k^%(w6~(6~A(&TI4@2HY?`RziyJ~Z@ z>rD6c+9o$dffF}<^2wD7QU(_*ha1D{^elUpc=SCTMddp{K2+%Xpm#p4d*KM1TzMF1 z$Y;SEF$JWbfV*_OB>8$F{rDeC8T}fQ#@hc9u$CFo`kP8g-k=xS)+GY9z^3@f$YBK7Ice z`}b|GO_Z;qrKc2nQRn0c0GhNC7qb4GKlqm>eKhX zI==sN*{X}()Dh(Xkk!gswy%2e0`tajT3QL2!}|Y6iVU7oRa|A@NC8UL23WLZwQv~0 z-wY-2x?$@XIVpaPq?nN=4RNKGrl^&s0XWitFb*bAd0V~Q>L%(|3iG)zNcxR!gy1Fc zX52ky9Kn@n`&MJY^Psq~on+=HgEqJ5U3~sXe9?Y3A zghl2KZYxjI+37Qj<50SZK?ANjoQ|x;1kQ3Y1(qKmg0Xpo5iX*SOb0tlRlRx`!$_u;#E7OydPrSdPk{gZ}# zF4UGGNQ~Ks;j94pMAG?_JG_ZgVC{L!)Aq!#N&=w z`{|@H^`XAcdFi5w_1>n`e45*>9a_Lp$x98H;mv_cMtmv*31P8zSKsK&?kbIGtw`G) zFX}YxIIo5+_~CVO)kYW9fTzvFHtR$Usdfzb)E6sU1Lu=AJ{B|3m(A)3xf+mxrJeGm z-wK;eqxRvDGK_xIh5GQFrA`JNL-cZB4ywN&(tU~5Fv9greXbb{vgeKV7zVavM^iDd z>)0Fd-#w1~U-0(|`t0Q|LBG+}LCE_zP5txT0m?qQwv`m}6YV07t5;s;DMrcj*p`+s z8T{c+P-Hn91siG+C=nz>ul0_roqWb7rA$A>hL1~f#nTjiP^7kr#~?7$S!%Q|?a>-5 zd-Bw$^sABcI0D(o;6sQ`{E)3YQ+)ho$D*ymw1Yr;Fyb1eBh|*KPfbb#am-N=aVZf0 zmU@u8>>w%3Sz8K}iL)Zn0R^`Clg2+LvR{Amgj1}&kMM~s+hU)#rIfoIv|+K2{s47S zWFb{#M^Fh60;C)>xVZ+vxkR%z0aYjk0Joz6(+>#qkUD8ksE;KOYm;xZfxSngE4u=m zEgC!C>qcmEwXchJ8hl9N$OYV>c%&ldxQs$_&NAR1%dbP96h76Tl^33p(>s&Z80exB zf2r4le(f5o>Qvk|dspBu+M>1$jIat{s#@5=cL}M(I|ejk?D;#XOb4rKo>^Lf+}>ID z$nVCIE+!iKbd(ZjQ+-F7l@21RcE~^)U%64L#fMf5D0k>*WXXAk&;W%w<)Qo4I|n89XwwZp(H$I87;*Up3>T{_-t9hf%i;6=HX%% zj;w!4KBu5abHS!(-T;O04>XqM~JQ^ogusnbaxWe$?; z4Su!Q7YEhDH;Od|WxLDh)ppsQsqx#=v=?`tP!8Vy^{zt>KUltf*>>jMklbV=Qop1}KrZOikz;x!*-62^hbxT@Gd^Y zKbtW0b+zqK*4E1thKB(cI=()&*-s@FxywF)WFH&uZhvXXaEwt%FuDu5hFL8(gKVaj zs&}`ej{}d_aU12ZxFcs;I$+Y-B|mOO8j(1gh6PO>G0Yp5Mz4*a@&Z7ke`0LPHVrmO zw#dt3K1_kMX-qx`ur&{h>O_$3A7#t{H|REX&V(ou=gH4^)7XTD4XPbk%Nv{pM z;92!3aL#+>x-9WKZhZ0*WocGzjydK-4tIWWMYIj&ae_B~S05ZBgk=qxn|rDcZsbHb!trYkaiob)fMeEiFt35%wIAUHHvoCw|vSU4rh9GHGFhlB|axWwH)~trmQFXN^Mr z?|$*`|Flz3{?Gnt2gmDo1L0_h=KbIOyx)(Z{}|a(f^a=48ZehvVMYQM95()LsAd1Y zy9xPX+N}JH*h9RWD*fW4%p1-`vD)u1`|VVUNBnkFFgq^)Zogpl_x*1p_~+U8dG>t{o-#w9gXb?c zo1cT{=dH`XaqAL*9c15^Zsvzg_f44X?;w!tAGQcq0-wp%N8i4EsqoufZjU_jEEn?% zkCPAV(YV`iA?D{vtGzw>QVVnQEzqd#7i@sBiw|CM({>a(l?RdxyO{8mk*v44_D?1* zYrp01s|~XrY>U3man-xrKk0C;Vbh|!r{l$NcZ!ZAxtxiOk6qDvfmE~b*jb+{Pm;iv z4_nM*UZp`P!a|}TP|(?`bvwTc5|Y1_x#N}S+(bM64ul59sv)EI1OW&i52Rt-T2J)0jFiwnDF#gjBVRMBCUH48Mg7(9*5DC({DO; zY137wid0q6JKHI{ynmVsO7afz&E1K;R5Oqiwznt=R_5z@%}oRtl|&u(0Tsy2&BRpNMZ#UAu>o8_!YC-EzM`)8rW~lCs~i6a@ve$onLSISBx=gi z?hBT0S*;Tn)d9$|255TXPd~qH(kuF48>y?64o*PjTEJb0P)v9nAgkFJ%^esUOv{@a zfmh)wBiZD$)xzyhS)NsH4`GpHH=V5U+vPxMh=UjpGD#D0H+^%^`n^m_8x4)b4<)JxquaYY>8 zoM_Y7QKRrIZG^q)G2nLlXZ+!yH+)X+6QE!VpzmLSzA~D_s*Iy`42@{OH%I_infBmh{U!v=I*aO>~Pjj z+*UFel9YSD{$d^@G21?`+^gWI{)#@V3v(@Nt?Xp4tx|MHXj zO9r9RZSg0786w;QpaGmEK$3@>i%pO4wnj=8n1SG!gK$>x4|4!KD6`=uwYO6CJIKUg zymWoma@GM%-Np&{f+jfI2~8>BSh^q7U8=QHCmg`Jix(OOG`Adj%FnM9t4hO~xW^^b zpE2!jaPfA?yxIBRU1Mx6%G}52652LTx+pm*_UL|ixs{&qo*CRq zYOVN5$~6GM9@Ztk5M(9C`@=_6nafR*FYUn#0Psa`R9AS_R8|@hrV7qdB(~UGl~*6K zRkCgcB9%u{h0!YxK$Ez$S}mJ~ML6)h(ywbPHQ&0ybcH6G#~x;LKG{jW!e0GnZ%&Lh z!veN{mY?V+402p`VY>tHjIM0@IT})UWSN8@%{Hi7%)SgzNHEeW5&?AziYkh z-}fuD!+NR$v%I8QxmFJ<=YIV>hX3o~@t=-hpXcj8wyu4SO`q4C&+F?ywRQU(oBn~FG*ZeZq=9cWY;%*cW`^rOtxIZ!+j zd-Ll4^LdkJf8K0`b7CV|H7d8Qoj%W;Qp%$aWjwKlkvc~@1HQJ;;fZbh1b*7xnV>M&QTb*q7O2@RhkLKo z-d?9klkOFJK;#UR5eRV9i?Jt=D@bJ|YFBU1v&12H<_^Tz85}uC^j&e1HJ|fiit&YM zbM%1%cv+D#w9M~M4(7>%VedC@`=;m0!bYEND2;CFuC~tcBoT?QNUWP%aNubsv;VTz z%`@y7tRbwVD15J%QOJ0V|74DFFt3GoGBg0)(X@8>){8Y(RZJ4K-8i^a(IMJf_Jpn=(ir7*p&|JRseDOi3S9qW|sM1n-`| zma+fpQ2r|>R!1+zMDi3evS_jv3>XoHvltd>OmE66Un|Z`y$xumE=UQbcs*Q9{wT8r zqc@GZC9J@&@SQ%&SbB70Hhh#BImT;~+|$M^oTtl+&U!1cqF&p6leGbrQdbKb!6D^} z^9xwGpW)RNv9R5Gnp)>AbF&={iMTP+WP$ zP{{2eTUF8R;sZw$vg1rU(H$TS8xRiTqQmfgXd|9s*w|w$Y8bj~~ z-7UiGv-}^^TL;n#tcNT0ooI5_4ZSZXW4uYNzKoE`hh(5&5TuDI@&c8MI3lIDL%Ip2 zS{LZY^fHre*Pr$*epVAFYpS!&&Ud%{C7{C`f7rm@9$Eb?92P9J_z=mbiuY-M;H;?z z9eSKS^FExp1b~rZCYdXyhURWvJqU?escDKelM}KVjREMvf-&n~4EMAHCE@kPcX2-+ zsw=WsZ-{YaO3RvKeNEbG#7;zRanq1PyJhL%GE>AfNOuOU(C;}1N5tKtV+xhZ4L+d`W{=Sjo4{x@5F zS)GTukN^bG7l*t?evW|du zE==X0JjKrflsr_ZW6v)B-jp2nH#4+;Jxj)pKsmijx)MFagtU#s<@gZt{f5~JXc%uS zVyH%=k-?36Dx4oG3UNs?%(ylBc;P!AUu{7xuF~OM;((WtkEjJF)`WXdBNl?_8L%)= zzgOlZSpCAF8sjO_;SAWOk%CgmV|`0O_L{abLJ!-{h1kRNx+)t=TK1W;$&_>N8(+SE zL%!R*KGaWHXKXoDHMr1sEijbzT@2HduDdJ!s@p(vp*~P$Yv=5hE@S{UtK7NBND&%z zN)hlL)&yrEj9Q6Q3(a~1+rHl3dBOAcaGgGb$PDewgn<01bLQ@f!{<_LwO!Lj;WfnX z1Y#z|9-v4;%n3KZzx!jV@P9foe8PeMZ~3gvM7_l*2%Rn&kH!2+s@^#gd!?$~KG`i6 z0B)Y!sr#V76o!EQI!}0)Q9CqSPZ}D&KuLXvxC9vpd-q#*ogv=PGIJ&I4LjzIL>E`pdO^OL9j6x&Y_|wY=No+TZ3I|l zM{RA+mFQtF`*ibIOz@r;%horjwa;tJfOScW*%4IGdA?#YpvE$f=&BR=(4^o)@lwaS zowenOVG0PER2#MRVkJMS-lLMYvnkf80d__)L{9u|?Q~vMZAbY{{fD?OmLKlxr><y@$%u9XyN5WzoOS5Y2f|1fM5bQp zW62+4%IW@2K1yB&M<496EXpoR98z>+>bEzxl)PaZj;hYkvHz*F@<)Y{J23UJK`!m$ z@7e}qznQLDtdH?QtTubIC-BXDo^JExjYa2oHj384m#V zuPy53SJw6cfV~hSHFZfC1p)j^RgR`ZZ;_s_2$bV{siyvjg zAvJ0=GB9Z50jeqh-W%+Gb-#+6naJNh(n7pf^&7~9DhMNYD?!#^F~lM@=%b7&n5I+> zP+^q)>-*gpHDUkanIH&>L*j*}L0`6;wE^{)TcF*h7UFegrJT79?fHZ`@COvEcg*k*iI8N zb$M1^vJ4U1|Kd(DQ3rs0(QlsBjrnV*XnyJxhn!z@`!DXZ^|vG7)02KRgdG0<5c*F_;YXkmqx&+M}8gwpGUyw81R?C{5b}EUIjj{0-yH*pZ5Wu zcY^vd$RINLwc)<421tu{sBgy3%2{u7gpx^ z@i6!hJjsk21xi02W9Sbslm2bM*&&*1cIHJM051&b(n^gvY``yphQGR`{>3fzzw_@G zIB+Pr$E8kZU?vQ_f2F8j@R$33kq78GyO{VDaL#*+>#?zx9w)&U!J#YJ>FE~~m@qO> z%4W;SORPHq%2IQJl8a9EJYgv@uU!>WNU%LmwJOy!tCWN=Q>P71BM>WUO8!& z(HY0lX#q;V3LjDb?A6AMdWr*E06T zWxTM>afTy^hS=ciOql5`37qC&5d|u^gj41yx%~R+C$p!*UwYYq%4;md&2d>(c|B%_;eC#%^ldYoyx!dvWj33> z*be0$t)Y6jk3YeUeaZYNV=^)7I%XOj&Hs}B?%A^`Ad}IlrR`2fifvU=W&2ydC9wqh zX?Y2b8Qz6hbO~pI0025!3m2Xap1*F|+1inCX;#10tarbQZAseRSYk72447=`9poZY zvUXL*KbzbC2EQhLm`L&8>EGm)I~79AGRxYr)sWvNm)u^N=ISOHZz-bkbgK#3!|-S` zO`gp_Mm`*OKZGBgGOmYz zwJ5PTT|q3T#Gg+8K&#BLx0BTizMdP7Hs>`#RYq z&mPLO&KLj?b{1(vhNv;<)6(sPP9OdI#f#UvN7Ahu9yP6Fu)eZSwkH{ma;Df3JBrXv zBs&*_D}0aoaCHK&_esv=iW~L*q9R4)$CR1i`HL1n5K zpdyf}l|iI9F@}&TgFp;XnIVv>1r$UC3KS@yG6^9dkO2}%lu?E-50a3CfS?Q^2~z?Y zykF1sd(MB)`Om%Qzw6$0S6#~meEGgN-}LVN?!BM=JWN~>`@*AVVYNX>3c__FsH}W0 z$2HMsgj zbCh@TnUYEOj|yc`#xk|i(Lr^Z^j;Nh{{C+|TFb`GDeI8lhmbYnvwya37!_loaYq64 z2}ThF(8eDAs8Bp;UHUvxzt7t4Uo&r?)q#H))t=%XD7|Nh7pfLmS78HP9d3A+Fl(@m zx+C7p=AGr`72WLN4i-90vNa(zH@8>WlYt;Rui`R#QVzFZ^gT`1O$-XB{h8rxpmbNZ zRPDLikZ7X+hB?I|km6QjY((GH6B-9rzr@!YvQP*{^WXKJs0ZoEc^6I%FDA+W$8t~)C8+^?7I@4qiGnt6k`>cT^BC;JX}JJH_h6t5Iv6q;vI@I6?rPZq3$%jk2L;v#$M^aBa*hgi|X> zZA&^R4r)ID^+fG%KHOTh;TbkpU4hVbFPN6#Yz^-JvhoPH%$Lb^9?~StisiKcZ!?%4MN`5QEjWxEh#x`d=tY zY=-_rvUS{FD;k?W^;hMeHaUO08C@*|$(@$DBw5lR<~P?>`kU**{DBjb>$` z?Xoo`9%$hOAkI}Pw3YZ6(<029W9{-fJ^6iCg#I~l%0fqAw`zo1mp`!AG~eU@l>2hE znk?AnAto9gH+q+&GRU78y0&DuKTUVMd<;&3Dee2LnIqNfBPc&ml^@SJX@k|QW!S5K zy%ePV1K_Th%bGai3vuwoNG_M|*($=U%P(`jnIWZ}d+6Qt63HZzWh2-P^Cl(j z@8sog+ree#WPFYN0q3s`zlZ$qI z{W`poliM}U#8B$7wH<}JkNs4?F@#UZQ!8xC(jzg5@W^u8iegqnXAKi(fKbO{j9Cmo z|0-st99WfX3h}=g`gW0E<$Z}VJbiB_dOkN!ZTI+DdAa|Xb6Aa($}x=ca@-FNwllb; z&9yZ5K+-FVCpClJqI0p(th$DLt5?i7H8$l~^)_sMED~E!Yjm+h2pEw4Fb8eK#zLuXLgVT7=|)&D`{ZnB@+Av^!;Pi zl&+@Q8+la1-Ad^(!q6g~*<}TH?hDmFb+9B!W_H!x52!HY-wASYY4-h@$IUhU0KRlP zKlHZX#(5}gi{_z^3ReXyIp6K^I2~Hj^W6SmYDSU!jzlKOB%UQAp@+SjJ}P{%lCm}L zi?^PfhxxZo-{@rbthk=5f3V_u21RTuis%)T}7?`~Fzb!jKzj>&E zD0(->FKyTjyjLBKw1_BsA{Wp4>PnP6==C}D%^TlUPQab^XpNOe8RXVg`W$xKuWj%Vss@vK} z?+zGwAaOqwq3=C?vQpn&45|<(eJk8M7nPqIISIS7(Ka`c>!PZtS$p&`=t%7eV>S!W zMH=QcsC#Y?bllq+K}{6|d-OM0RlP>O_>k$UjvF_JjD<12=N@T; z&|v6|bN}GNh|LHO25*Etyty_qVDO}-a+tM5oMBzan#g}%usx(e(S43M64i4*isa!^ zcHq=Lr`Q0!}uwjvc#oCOl#(2ic%nDROYF&8}MPt_P??{nDZvA>w? z>l|Q?2}fmrRKTcL2Vri=T&&cZ$TwX*bgQ=}7#UOPs7i)u%Lyo5j`Q*2f#jV8zT3VZ zF7TK%>i7e&V}a`->}_ z01#yTurr2hFCPG{SVjPAw$HSI$#iUk(JVi`Mq8z0R@4ceoGU5I0MEeZHwf|?09x?W z10>c3y+Z>tZoAW3D$sYsY=Voo0h8|_P@VbvO|GL?91m=xy1EUP%peJ74C_+R`z-k% znq@v%pt6XE2rfY0hygu~eFZ$=tIv{tq6zjuvnhi*u%Ida;+7rmn z3x=ko0PyYAFPMr9%m>v3`L=|0&>>7yx~vHEAxr@D=-L9&^VPrK;Jl`!%Ygs+lCJ&D z6=DAK>Hp8JXoM3)l(-X?CnXnQY^Pr==`=Reg*dqJD<&T<-mAQtRwer!uq&^>1{e>f zfC}Md3_eA+lMSE-UfBVQmF+42X$$og)8^khr?(pZOxyq24FMj@e|<-`FwFW_fmR$# znm`vce;B^8`592%ye7A82daNXK!6F_&};yf?mq%gdr1ZKpEt$aB_`uzzK8T0OSdI# zbhygz`eP=3!K4a-%)(I!!&aVv+J>3X!ARq5|L$-256D6!=-+u9zhC-)o(8`^^?z;x zet+SApX_V4ajcG&?TOUJQU>L%$naS~Vg(Z>6f zhUm%<3>LerSY!X;!R((Nai6|DA9?)gXJ54+cz@tPI{Yjgh(@ddVqB8^zRTXl#M`f5 z+uL`3&zNH#s@grNXH)#erQ*$|o#O#*MvPN11o`ZX=ZSpli=XBj{#gmIHk3I!h<7}* zXh`*KY)Dt)gdd||(+4dvx0PQj!ZqM$Nap(7cOyXu6GRB@nA{#;cq3`x$5V6qpbMio zQ@aVK0V4Qtg}D=AW=q;P?<5snGEN2nZ1EXsRoLfx3iN1Y)-u%mr%}2^g+)kI;U>_c zG&8jjo;wGHN@}K?u zw+ooRh0cRzRU$_P*OveaJf=z(43}-GcS@8WZ4_kvq&4rjlzy?EOsc+zNd1KyP~s+S z1L_FjK~jC}C;MaP8Cwb_ky`T(Lp<-s!Nv5hoNcl5h6<5eTX2vT57!?<5wPGEvyF`y zihUaUQXo;Z8|buX(qv{r08aBIl4SU7OTj*Z%)FHsUK$wg5udWNac8<|L)4k}YwGx} z9f_km*xnZ^=%QE{r`anhK!9l)v54W7zjd$-uX$~-zS9l z&}xDVd97}-b=J{*ht?qvQ)-=?^+1J2kK1?o9v$_kS)_bS!*D6sZu(^Ml}RH~zyxUj zTtHY-bZJDE#B5mlB&>Pug*THVrn;-wgA&P{hf;TeXws`#m4GV!umNjr`xzWo>$2=( zRXzYNopIL)n`SP_IF01S8y;fQ30}B!{SY4^}?K~@vnaL7w^c`)C8I{DhK&Ea7E;4dXulfq+fAZ#TV`jRb5ymIF(+Nhd?05`ny4ur?|sea(#$uENVmN3*El3KDsx(VJ;YGdZ-29R@N zr4Yj?pb(nE$Ipg3ooMpt4kLc;>A1Fj;d3_1(47ja0T-BYvfjy$-GpqBYHz8_DfgEI z?U~{|RAcB$EC7tGm8ufkgCv0>NFh6l-GZ(#%Bt_=8**u?VvN8E@>a(yXDeceQo9e9c*SQ5nre zy-L=2wPADjFz>l1g6#8+Nqg`KrU5eypV~sNcBeJsO@R)IDQhP10_ima1g3MBy}qI~ z3f%@OH99YmLA-4T$4I2bVGQHN3QavCpE+sa9+4=IF&cP(C1;?g$MLvV+Z-sjV<{+u zw2fFY;|^iK2%`>y8?ToU_ub?jhgMNpmoH~s`C?=i8lF?lUmAW0GEyh8T(Aup@RH_aH!R=u8i z>c$C9y~_V`_=lmDD#d^ii@5r3*WiuKO{S^^qDL+>53?Ti|4lrdpx#6S2N%sugJHI7n z50ZkWH7<>g5n6y|7rmb>W1>~o8lRml^RiWbq8si%R>-1sAuA|Z;R4dW;9ck>^Si4s zRI}EOhG8(1?)q(=z%aS;ml3u`x`)K7jl(twtT7&&L9uGOIidkxRjuW=Ymr4_^KWaZ z_4Q;b>E%8BhgB*rkveWeC+t+R!*B4Be(%LOgoT^9m9B4)$b$|D|J~k+UyKGmf59ln zzP;@la8ea0sQkw0{4WVK9OHblDk=XPAkbfr|0lFhM=Q(?EfN)a?oGIVcPQI45_9ZH z1vAcGR*I6%LimFip(*lba}P7)r%$&RFv316u+!x-^N$L9ZP)T>A|SCya|Ar#aAVSk zBX6nqemsT=0AMFkPLdUl)DiUQ{w{0APYbVaY*#4zkDnhy!6wN`~PEa zQxtIc?TBw9dZ`prP0}~ZMm}z%;S%~)uFx+oa%NqfTJX)o9chu_ z$4Qxm;#?(>|L3BA^v)uH-owyY3nQ(oJ@}0K)-FoRfj<))f_xkxBN_K|@MWKaoo|8*jM%t&)6@Bwr~I&CnVn06mq_kBy~H`)35P!<0`;&M!^3&2CC|2I2UxmZ3eu;l z?VuzG-d+6cADr-S7OJ+A0M$5EXJ}+>$R*dg>m=^i+1^tvTXP1Gd|9Cij{o3L9ksPA zN21~bKfh#v2qAIIMuL;iM-9{e>81V)_HL-wUa7*^pdz&`DbsJ=*(FcI)*w*G3%OQj zA{+d6*QfUNf6W@7ti0w>L<+ccBh@`q7joj5Y$`>(OFGx{U%eow1A;d%D4I5;(iLOI zw2@uk0s+!LatW6`JWoe|PDkkkpW%erG4JF&?p*7UgUkjZDzEiezHxUCUiDD;^bw|f z`l>(gln-8d)PVrI9>;=j_SN~ifFc7NSue=YD`1B^^NrWW&A*nf9r^4Z4p2U7Ay2;+ zl^L#{{HvN{^2?4}SOAC>_?}@XA6!G3K_EdN74#!mq>`|Wi<|!8DA;JVzDvN&P`LUJ z-n;*%dA08*{Pedf|NorEndrOGyAxG{XV4YrYj`R#Hp9=Z8lMmSBc|kEgKPXrn#j$< z{85M_#$^cKc*?bA$W-Zg1`3CJ@2*4gV-b9uC8g6rkTKlce}SrOIJ@>lR?(m&`T7G< zlz)VrIe)o9c};V#lXEmBnbx{&QeG-?VI6@&|fyghz%Bwch7Ha(#ur1Rb~8O zje>a<+sWBb0Sea2&mb$dw(@-+6=dzu=>2QjIeN1Mm307G`sYtnK)?rHz{I%MA;w%D zo=j+=DOIgFTIC{Ovm50cljvUbKrF&+S!2n+f9Y0~x$2q3FR%@AeNl@CQL) z&>jyu)+69$Yp8)@bO=`C0=BO5?bLcWrpr{$i@oo<3G~`&%UJ=C;NedCKeN<~Q5q(+ z<*k*`A~jpm>`m5l8+x;gpU4)?(fGkl@9EJ~dM%i-IZ4Ycae)^>xN=EqNfpN7YG$$5 ziQ!>)&T2rhw#%lJm}i)gpj04q@F=K#-;$DKL25zjT@Zd3wMgO8UF|ZLVjNrqLlzgJ zVkXybCI$7sVTYI)q#4tfjat{KfuebU`XFCpSb9#i9#hyJ`JyD__OqZoh8_K9x`yK> z8)%oWU%YWck+f7o_vAFEj8R@ewym}fH=HW+Y{qC-SCeuiSIAVe%;F4c@8{$8hnlsf z;hA3-PQ%ZKZ=%P7Be0Xsqird>BH&G4;G`xGuZ#&_p`OJB^E8LxwcH15i|LDPUFwT+ zXHcBvwDg?7Hzm%}dot!aAgb+}B-aXLo2GnA94y4c%n%OI&a=*X?5|k7_g1HRIPIN- z*KU0zg-yXAUV~ES$uB~S`wc_tc%$E^IqELiO_5TeA*mI2On{?ENMx4?F{_-wsWoq| zierdJ&{o{{Bx>R|177U#3s2ZNZ&^&cZWT{+D%p1OeY03^ zbk%rYgi{-b@lcc18|P|2=-P?TncELu-`}vVcJmY_vfdpU#;3rY0%%*L8pUG~hXzK^ zITHn!BcpaW-aZkN9NRfUiM=6vzv$yDo8dQ7_~_v`VyCt9;%gHHdp>Ner9Wjgzxa@B zzGh$G)>xPDz5DeD-^c9zy8ctinKoJ;l^jZYC*^zpG=M?}1CpYIvAF{KMfnlvx17Bp zh$L)6N;8pyFS*AGQgy&^{ZeCHmQ?_ycV~D;fQW;qwM5glqmK*TnVZuD>4==H2RB2l zFwNv0;iFDk86hK*XN1%SLI5qSn_lmVK*9l8dHZM5F9jGtU`}OcUQzBKlgn&q7h*h)BycdvM=) z`ysm4Opaq`g_<%+Bh~}m|2b)4Y_eB9qMf@iD4usBSC1O3>SGV*EHy8fz@df8x?G&d zJhQtv{20+(-$ifX^hX8Ves0Q!beOg>8|sxkp`75QC5Ca|;2}E#IAe@Jd!1IxS=}^y z9pr$$PTC2}$MoUsZ0tip$^usK{uP-{#XSqyVCUUNgoti6gC4Hh5Zw1?=gKrIFPw5X zp-BmovX1UUcL=;M3U!O83ht7pNe^;u1~A-RqkKQX;=>}w>wrVAUqIJWD-i1C>&&bf zc(!{ijFKB}D^LaiQ>`4(JV#k-8+ij4NL3f2yBYDhdI{QuL40n&!8AJ`0@Y<%cWKPq zW!XH)#Scb7e86JHre2xd7N10d-77e+2kp(04p$b2YB1N+4AN}eeY#UA#}*$ucD|Jm zT$>@apk~w-ahvj<0BcZm%$b46$_gG`Hk9dd#gWpT#_lijzveeS-mWg2!8X`^jk&J8 zZkDnGGdcrgM@k*a!xv5g_pfn+h7-xNJY7nE!uY}ytT&G!SCNWNBP;Vs(9(wZFQrc9 zrsZ$CyEz|tcdjwqf8jAnGqd&+v@s~Rw^`=bd{w}UCzLw+o^sku78hKNzUt@a_ff$X z5&u&%pa;)t7*a*MOU%)ZBbXbUyXyRSgB|H})Z+usMlF>!G?@f_O59*c9A-YR8A>Nf zYE5H~OB5o^KnVq3SUPd-^rC+WaoRuc5lA;?KRFxHzlahMM_-J|dm+jps{-|A?l0Q} z;It0zOj=|SVoPdR2O)b|m%kA}nE@E1$`K`kg1dP5=g`1gA6#@hObyS!cO|Fvra6jS z_e7@KFP}h;w-ATdZ3Q^@zcC;8KD{YwbvmU*qu#L48DTc`thzco ztZGqY8$wb7npG696pJ$^AzA?i%Avd{eMNu=0}oQ43p2FNR%RHi$WjT9ZqZ5Iuo0fd z3H{jRa5^Or=r3u1rK~WYrDa`D6%48E6)!S0AiGJ7!&hSVUBcfs?&MGyI8h;EY@I4H zz4Lxz^gjQ2pLfBz&0r;qDZ9ZlqNsF6{oo^!}xlV`G)P<&x!2Ns&1xp zRu_x{4UHE4WW7i1`YNl>>qFF~8ghbNZJ%~JZ?Bz8+hDiE5^~CXeA#E6x*W8gjm;Hi zEzkw?lUtL5(6+b_cRT~l<2RvG=gcoM=CqtP3JB2lW6ifo{)#i$+pTJ%B|LXn>YGw4UvAyv%~0dR*8-@+QQb9Xb@IKDOLAwpApjXJ6>3k4 zTJv<7Q>Dov1!{~jf5=W#|B>G2!GK*IYad+NdM5q-a{0&PKxUQa>xb;_BDk_FVUn+< zX2Q`#zU5h*{~j3J3vV2=BPwplEQ_bMK_n9&pNpTU1VGkqIFSo+6?0!pabyLe;V&W| zE%;veE8V&gwU(X5O9LNog5Dv10Q%N;AS8{n@R7&P7Lia#psTpzwMK!{FO zonyYfJv_ug7}v zm{h(ni1q!n);60)Gm)31YSc{7ahmZ z=+iI!3`Lp5kdy}&qiwXeJyWfS5UR##sPRsg{o&Rh_t@_1uSn89JA+Y-M3Ssj$ikct z1pb%4gn;841ePT$nwZ-si)mUP?lW>LT(Ky=7JHLnw zi!owHa!LLxnQn!TfC6Z&(3Q*+tMHn5@5+%~X2G*{EGl5h%J zRRitm!#`=CdBdRq<}3m5XFEArK^|rEklKW1UU2hBln!Ix$OutAZ>0K0u=fiVl*Jey zx7W$qbN2RuB!gJ*^O6ieV3ia!ex-gB#E+vE(cTKI8{>K(NqeNo_d|VMCs7Z1bka+ zc2R0J@GnW7E7g`9lb#e9)uOQe9Y^jj?u3%P`=Elt|}FB1C{7OT9^{tZ&kN?k+#Th^9V+;cig2SH}IpJv0A7Rzfm$% zn~M_pq)~eW@CBUUJy%|Ptvn*E*<-MzCa?4!3++^HRes{c2}~iO`HamBDl3SMxXf7# z6Gx*!_ntW^HC=g~Wy10Q+(XiebleZHy5yvGYpn=V@+SE;tr&1yPVfXwJrfKga0)w-D?Jx$SviBDdws~2h1R$T%XLsUIh*95 z2i%bvxypJBR?k)p;VLO|e+8n{3j9_0RyyK#fKjf_K*!9vlffqVN8JnVMd`soz4UWk z)>nk`)z&hB(C3k9^Dt^+}(R0JqK$`~FpwM#Qw`4H+n36!2<&7-I=3oOiswfprIGirN?Gy6Ads)VhB~d3PkE z{OHrKL$P1Pej%Q3Rwt%<7X_GE$u!IHOU01s%dUN) z?+Uh4zeLD2g?}K69+{rf40lbiC-hhFzp__H>FQ<_rP;)yJJipyNb*Qr6w7SuLvcA0 zF$rqJ7HnaPvn~sI?iYRNn{!nvw5x^zAceF;a$yFs4GR*;W&+Vv17UdcFL>oI@FUcd z!mz5CaMirfNb@n3n_TDABC>BXEutKdrk0x=01N9$NapqzzvtuVz%RpvsF+}?3}XY_=P3Q#4f#T zDKty$D#3r&IS<0K*cseBqe!r#qxqF|7jl27+Av{D!uNwghH_E%t!h%2Ue~k~E252I zbGEjE_D3X48OMiwz=2;_n!gYSI=V#rSB*>|D*D?7Y{9o{-drUPUu3#N!#lf(WV_hU z1Blj|)q^2byCYK0E=4$$&E;zfH{Sz)J;yNUI>5ei=NhuJvxA~U z$RgZ|Jub@!+Rc7~H+$&by&bzF_Gp6VaOgumW^fv1bPkbnLR>W7&s)bNEcR{Ddt_}9 zv1}G)19Uli7(ZOFymGB@?v_1z5ZbHRS@C53PUNYEdX?N@qzjB@G}n)M6h*yTId2vf zlxkF6#))VRIl>(>^-~qp2F#niLm4*IiHup!s7cfs0^k#ikLdU7t1_4#4INvJZuHO$ zblTN%TLT_bu1R&kV3*d)D~I96L5%*SiTEU~nw(ttTrev+27RVy$(FE~Ik>=d0;b$B zZ=Lv2AzDm}PYKy+-b}-T9`IMK_64$)i7y9@8crL>RW5!tY&m;yNPF4ZH>C^Kx0n*d zW75XtYp~Kbr(I&`L`mvo-YEydFwdv1o^%`$H8cRH-(Ks`e*9(FD|WCMCnFX{TdbU0 zc_H8Kf7|vC*MV=nX?-NU^`Ybqz~DA#lbQor#ms$Fh~O9lUCM#=%-*pqn`rosR^Q2# zJ+s=)X)wp@YouM8HBQWI`5>mJoDwb8u zZfY_2aEi-v6TJWFIZ4iJ%)Pj@=!Vw^7u-ncLLDzm&d3cm;&aR8da}aMj$~PM{*4+x+*=|eMR5uIeWv8KNGI*lS-uif?qiNhW6hkZefLYh&R|+Gr*kX8RLeL9~FX8CT(menoh{E8xD_| z>mCFcN>SU`9cA!{T$g%*r3LAsX|l1yy%?xtyp^hkdjkPcg}+b(Lw93|obXNx3Ef3` z5463?2QYiBcZ*{|iXU?D>ahhkA`iba%OV@*4P}ayNFj-%yYPveW3mysy3>x>A zD1Gt^eyFh$ERZqWlWtW}5a{Mp=aSx?UHBNnwk7rWEsa>w^i3mrd8Y($2l3%4%`JmA zLnTAt#^AH0QA4RDDY^_P?a)bn7JwUjmWw#UX-Jt4EauV}(UZ1u;5=o=m!n@HG*vZ{ zpl+2bOlZnVbTT`%AzA0K4318V&$FvTK;!M=F;8m?G{D7Lgm*>9Xe!7eMC95jT5#%JLe?O zGu(g$$s|>So)g;v!J%z3$x)cRyFdx`Uiev(d@pFXy=L391GZ5fw(12&oN%EDKQ%jR zdYl=Hq~y*`p71nOW})^F6{_#3?J-DzHX5O7 z;n24%TX1I0vZUS_Dnc%iI@v$_vyb;RI7F7Y&#L$2m3svQ$K-FYJLgUo;nNFP-Q^q# zDue1G=P95ro%115N-C-3<20iDEh0FAjQN%3?|=4YfMvttSbau#NXJv9Kz+!iAzVm| z#oUH`PG3XcjS;XIq~kUfw=c-g4|1&haY;ExjI|jAZygOk*gF)?TY{54->FXQsC>{9`?Ef40MTM1ZtA$w#E;+5cBR|CjXu{=Lraf0M=@0ONhOj+*FwI3Vw&smYC{m-BoInq!X$ZIke) zUYwB7c!;A#0 z+blII>?cQP0#@7yp+^&W((~+$(zoY8S)gBV?VaWOVm8B%A1H0-?4_QR+RE~uiCejf zP8wZP?qDko+abUr?L~wUXQs}jtqGAn&=g(M$ByM3e^{3&c5S-C_<%U66;ey$ZJV+} z$oC=&64iU+1cA#}YqMPA_vLIs+Hf9P_mtzbrkB;k0C#lWc z_da1aQloFe^F`8(>0SiMSxo2gDnL)bZV&I7cP-hER;b;fd2FG4dt;Ii?ehYCa05)*>- zaNV>V-8vf8*xszqAG3e=E1k>ozS!7MV1VvbWI{hG_%nh~8v#0*f{zMJ6T%44_}VXY zDG-=*OA;nh_C}DI7yGPLW;dn*=8G7J<74(PwO2yH&Uei)-^qEjVD&|01rxi-q{Vsd zX%y-fPHtOKJZeg}y&!L+(2mQO5SmPC4_(HXHU*+mZ!RR!yKmH0pqlcc@}aj@cIiXz zoA?pP>McO+Q61=0w9!|dtc zn;LLz2$B#x59FxAtZ2sF!w&_bsqmJn$2nQAa9c1BO(L$y_aL?5>Y3O=24kI!&os}! z5Da9`YuW}3ZgP(C7z0Q+g~UeNOMsP18)V-CAvyTFfl%Ykv+O-YL0bX0gO@8)c~%F+ z5GT#i*y)aGp4KX2a_hpw3C;sAD}gV&`1yw=v0ihI3OfZ9pPc`8v^!1pvEA7D^;mei z)KWTBwL4#1P%st#Na`$9n^;T5&A*Jezu@(?5EDN$>e8cP(W2^NV%pfmgR@AEY%dlQ zlUOEI7H11xIL)Bif|Q2SLcNPZFl3jhS%0YGmbtc(Q_jiH>im6@3m^Kpt2`SA!f}>V ze`)ls2k13sllTru@pMUq_xIHaP3+tKml>aZS-o87KXW{8agGi|O{!Cu-O!o>zl`T% zS6hN$^=I|)xJA`v8=~{6o4(sSwIFN64n5iG`AKVye(P_f=ZEDW=`nF0q_zG!O<0w( zumU^#S`pz#T+77003#8~i_+;-?-bD^`m#a6A|X5n-b#C*uFTtn(tCu4SGtRXU4%u9 zok>P1h>|(4{S72!7s^}Ph7(8tR1J#K!@F@yyvjt>+pJ3?azhUUqfvCPaFAzdtNc)B z7>wDD%O|}6Uy1Bzj9;jU1QLvtA~z8LJLj0wy;ul|xzyyPIG$}u6xgJwC3%FAEq)=% z-sf|oez2Z1}2fgXAk{ak@`|nK_zsMnw|D~)-qXQxvZkd*VWNge&>hH&9o_rBOuVs9IK8bo~V$ysD&?7|6L>->2l z#>^U4161Eq_1~`Q?=ExmWXPT);AUniJ;aza+k#1#1wPzRlY=q_8E+6*tL_wS{imHg&?~UBJ5pPgicor;iQS)}iX##dLuUb8 zi!BTh?gbL678j3;FOT$`$^+TfQBPa2#!qO21DR07Z(#3mA-ow?B~EWAF{Kd0q+luoBzTDV(kTj%cl8ld?at&Al+8uYd>f2l%Hef zqFLY8*lL{5eYgLdVONa)l<_0#-u$lb#ks*##@jjK0%-1O0#Xl7dgvG5>Rrn;=bK{+ z&%A13YY z(qp5zzJUf?->&?NS^mVwa@_H9X4$(tF+^rEB*wz;=~UJ%-nrh8PMiR&UI6y_pIIikmX1J& zmi5;oz!Jo?U?*eFeBp&o+V!`;=#B3=f~)~DR{rXX&SxGOcWwq+ME;nl_+Q|s{2S$z za)9(~Ntp}?Rso|v9WMXtXrD-&g<(ty<&XThn-2BLEdlDvN`z-gsqlW2 zhjrua!a4Oi0+p2Zjo&hI#%(G<#of7tIbFquPgYg}d-Faj%;$_IISEFg!m1>ywi~^y zt1#lm)dqrlp}d!M_#FbVldtQCET3Bs zk8P5(fP_JSN5d&k$2-lJM)CXlL(68!!nnu9sBk1gCzIpNLew|(QkK?{^%%*q4uN@s zafRjSXSn?PRkQ3v@I5e3FSutjI4rpYpMJt)vumE8MAoW72f$N0;|J+WrJ!)s})5@&KElHEJ*@TizRd|gobIu0IVJ$yf`YpGa z)7)%661!J`qdjbZOlmZmfp&H%;Ytcnw`ZVpUcu-ATTnNS)Wx6!7g=eS9y}Z4?MCRs z=hDw_pboY5sMgh`Bh0hZD}h3)EGM5md5RekbOu5(6l>^kWrK#OgT$wx{Et^Vp}g8p>iq3TUcJf)T7 zKQm5is)@{{qR5>bc46kh?wF(J(76G{;^&J~E_I|nLX5?v<&*o%0TdcV94b96AkK|C3HQ1MM2+St$1m_J(Dg?9X#{io851c7%=AtP7h*V9 z?`%E3!txelTg|L@_WKQ0ks1QK)K7oDp5~}+HvHIBuIsmsXtK0_7V(@vnK9kkzV;|Q zxy-$znBzVhuBHX6d@DNEU2AZ>ozlImdQPs3%DppMigwSjI>vuZZvD`cQZnywc*>j= z=M(SmolP5a*Lj?@_4rx2s(BI(F9#ZCznDo_t1|A9zXqyNcQFJsjUO2c&jPs20BNAl zE2o;MN2`U8lXez+H){L&v$u_QrR`TC;_ny>YDit4mDmYQYe1NW43lopJc>Ml*lWKz z2}Q1BPFK!Xk+CE>Ky|Ao!PB-Cr(C57Gpl{uQpymHhjWct+1~MA`eb9q0<_Z-x_+?Q z@@ZE0e>-HM!=jxqyNC&EDdKEgIUB42l{uKP089$oW1#A&P_;@gF4mNYyInZ1#z3<* z9}oKthKyBabxXcWCNvO`G{eiDufhvk$v)u&9uBvV#n9Gj=lbMAcU^yJtJ@fD1WjYf zc{fj#V^72kSN+=H&W&8f>~_EfTI|F|UsF!PEoN-~m40Z9m|xyqUDe$+V-QW0jle}^ z`D#E` z&bh4D9Kp|q0PN;kOaZR|0feniA|elXE|2>sQ#AZzmHJ#gMo^jUj(*PWIcBJADxU=# zp4_l0jEt0=ghg0JL<9Zusd3-FlMgU|J_dSAwsYEhgi)ae6-Tpiw;owb(C(KVFf!FxDg)} zB4b6OvQUWn2eR!0JblZO7$_?zX}dE`NPMD_4~_CdfRfX$i2KbX*EI9Ye!Pi&C@3osCp9r7m-8f@0E*|kA&u{aiV;`N3AG9) zX5@5ZK??=YL@?OhSD0c(*vuRX0Xt8zK4cf8>wV6_oS z-i(Mw%pG+`5J6qKZ^I1+Y-f-%29ZY-(L)P%Hk_(pL}a~Aci2*Bou}0HvcIvXz_gsv ztaEGqHZRS7zqg;e+4>`#oPOYLcl18%ulpA*oo{pT)MP?ebSOD(5TS2?l)_no)m;9r z0Qr%DYeRTS3jQs<-a89n#H;A!M9Yc;g@9mb1jwBx&Yk4%2~e$R#RY_IAbZlm)iv#r zgdkN!45y`{qMXA*65eO!m^A`88Kv1}@r^6@ddkz*5+CFA4t-DJ2G4NTh|r|6NZbZZBk!L zMob#&*;P}Mngrss5kI2Ds&#;X^{C1wnx^`sK~4otC)3sfPj32U<4nU;3q-{zjejZO z%*&dzUdJa)4k8M88{ghMm@ALOBK=ac#8enoHdio_nHy6QarhY}Z8fz@7vaXg`HtYv zWi6Gr>!jIL%XpOBFvcr^VSGWffQU3(8qvrLZVq?7?1;3&!ocQox)t+1MfMiZ?tW2p4=nrrerQ#Ve^x{!QMOB7taO4RjJQqbLWGzZLrs1Je;Z*he zO+?I2w1)4UQ8M15W)9jwNG^=?us70S(B`IOQ#M%n051CQ(LbC?Jo4h-W*@(|EVl%f zOqEEF)znBpTP_^5O2>Nru~PG|W?vqH_FHaeNIu+e-8ed&k?mN(EL`4X%=QeLm(v5# zXKIE+L7zpW4W9kwbfsqiX}4P^G>93WoM{FqED}e1ny0dwDaQ9)!oo2-S1)m4)+cU& z)hnOSd}_&Y8lqq{o#=GZ8PiC19O~0r=fV@&JB4k9R`GWyWAxn{DN4rjl`B#K-9~GLN6z9AYhrr4YRdbRX=k1Yx*nk^|9n7F2cXRa1j-4NXz<#eC zR%*Ix`qg&*CTV#2T)wfG@Inli?}52Cu@Cfx99d0jekLW6?%3-jDj+M^9u+sHoyWU} zUAYBqw)$M0ngl2lxHJkk&pco>(tjN>|GL5qcMA)?V?!x+W>j>R)67LAXE){8=M5*I zvZs8oO#gCyV)J+SX!tI+zrGm>XP8I*3aD=t%#Do!?_e9w+47X+EJDj70WRSVWNm9S^wJFu2#!XWYh$V68+cz;_WYcKD1xF76Ul+y_lwhuV1vk2%^*_2jv{Ke1mSp9?17h5@iEf+(5FG`4s7avE}~Ch;r1<0QK1P8|tYQAkd2G5^|}Z%BQ`zQCkWW-jz&I3aGzc>gM8A*Q_% zewr%uoyp=x5S$bvZ2TuiPz*^&ou99}j>OU$x^-#foZomC1!8zfsK(n{=|9Ro)<6^# zH}d18rY{%_IKUhfurF^a`yf)-JbkC*eUPsoJZt{-4^|tA>q#(i@svJjuLer-i$}re z)9K%l9dBF?P(3Hv{-(NJlDTvZ@Tvg5-jyGjOe$}XJtgQvnX(@uuImZ33M!R8pK_6e zZ@}~38yK1}*Q3!H*~E_u6)Y*#|E1RZbCC(n$`8n*7UNwJzc@Z)29S5Ama%6qv6E)f z5D%aJVoT`)d_HeStcWcKO;$OT{iT2G?|=Sx^povKf{2%aIIt3NWys7W-88S>J=Hdo zw$v*t5S3%9#{n707PtriIMh7-!}F|#aLBSyO|}l?8DB^=d!^EIK#wyJW+|LM8rvTH zQ30|W!zqw=ru^SM?nMypf3)}IVNG4@`n0yTia09@0zIOr3{quML1^_LQbe2*NDxv* zkPt(PK*A7+mLeq>P$)s55M>fcLIe^51c=HcLqujt2!S9VfrJDE0vWuUbAG2+&(xlC z`+J^ypXcxo@a(nM-g~XJzxD0!d*AoF#Aaj+3O8pD`ue}dg9a#6r76SVl>|2~G*OIUU_32zyCZL}Sq&sOPz>`=63g)B57%=h9#Qb9( z@7NvBXXG70Cs-g5E*AXo{QM z0OZAXP^B8Y-|(wdj^^*R{op1w{&U{noxJ~y%imwr|Ms|~w#t)+%j1MmUj?|YZ@*e( zJmq!VwJQ0=IBEfUvLBMB5{drtyS_H~nR-PBR%c-oyqr4EqRHhol^HGx@$A9m6F@m9 zL{9!cdTVZM#KYb<8@cb>nveGW=e+-ChW}5F%f~h2%J2bPpEWh2xrUy8O`>_~wjHn~ z5^+<2)&;bCnKu5>dDrDgBTaSPZ4@keUwr9Lf!1I2;Juw?dxk-h2a%ZKuycabDWIAS zO+|2>77IAHl(tl!?k;IHXlVt^D+YmE1@p|gaVRNHt l>c!^|+LsBDu0*f%kaX*2 zTL4c0#AI3zrAa6VAc9s?nS=svV?6t?+{tGI8PVh&2kueY2sJ&?r8))i-<7!Tu0%F& zw=o0)a+wk(6=JS98ioeQeHoxEHKV}WBfyp1RP5ds=L)M;F~54)VQ8jys-&y;5a_?CQN1c-~94 z2I@zm6xUqMF>AwJ{(j=hK?~!>$nr7^EQ=jqE~eVDZywpDUKR691{0N+Kt&uX7uK}P zPdGSHtb=U5*LLkeRvN+tTdV9V?D$IG6^PDOS*HhSbyvM$KJqT8%{GiA_YYqsP9V90 zG2TY?G$N{pr+(417*H)JE5E1S+yyYHgs4q$D3ZZw`1vk$(6L;!JrAyTg^=UF5a?Hr zFM5L85~;O0_@nws!RipN$l(U&&E|@-m*Qat^GDCB#dtl|iH_niEkHHBw~~OGza}Lz zF(a&a=u$6g+f%)o*{^pYUdQq(>V85}4UGW-x=LaV8m}>f+Y;R#p`Bn=b8g1V6I#|a z_b_&jpx-#*L+{QwTBP$e^ST5CLYbvqr}_r~+;z*NSDj_VbgRJ!dTe~)gX-#;Y%71L zRXpgjO3+df8$CZQq@B#UVaIZpf0i5YU4TBAOo1~Mpha3~0^T;d`WtSSa5Si=S z4}eTefZ8X024#&gp~Ay7-nm|Fr;47HjG$pgC2VSuW)nCIK~DorFR~F|fmz4$W;klE zNRzsMf2~>*QS{X3Xg#@5ggHWK9F0L{gxfW5`Hnmz@xhD!iP+TdfNvyU0&oTj*wPq# zNefz3ZZNZsvFrwZmFTTBOV5EU1&b@W?=<>$0Z4*hB-xhZ%j-jOs#ySC8owXw zFcebym^fqR2C0s7Ek_>Jce&S-8B%(p1R%{=rAQR+_xhDU#s(HwtKDH`cJ+Kmz5O55 z&w4`->$rHb_kr@_f^9ek_iDYzP?^)9LfE;(Nrii9SxPsu(=cjPKm+^J-G zGR^O?>sUYv=;qnyJDNH-ySG=>6g;>p-Fiu?kPOJgZp#otAO)d=arAbn8;7?TW%j1FkgD!hiT!=;sjQ}XQEc9rq}sXQfB;| zgFygTZQ9LnFq%A#C`dlM3%(uNKE*~`0RcX?IZ&Z&M^ndJ_ZDv2kJH-ExZ?>)388zd zL$4M$BTx^h3R6R5AF-V6tDsN~I13ou7X3rArub?@iqfNd?E{QM9EsCePNA|~D-c@{ z;kjxN{dNlvK%h|FfJkhSKaFH}7RXUDInt`pxv1-dUV$E8)OsSlg}YN7GadNO*5p7C zvh^anL3|$cQVA0lP6bvUD}kerT(ri_FIC^+O_Vy1nWujTU~8(uTvjF`s+Ye|cxG&^ zkEqIIoV58OA-GX(#gN(Drd1!Q+M0zh`q^}DWeFkXc$Eva2oN3^Zv`?XJS7M-%qGma zx;3h|R>(eGWFp!>bqY1z)H>QYYxPV$i{cNoI(-qWKkI1a@nwthNmr5PujM%YuUmeT zRkdg;_i0i5#?^S+Lf`ZU&bjck=@Zmhv+(KqNwOGIKd38pCA?kot)(!vp|SijWaiXk zF;M4EX`7yk!J(`BzJ7_-6ea-?F~htt``NneU0qzi>Tc?O>-K_0oM>#88LbFDsAP-5 zCGmDmotESot00HZl`e&0na{3O)$OWk%nT38A9T?XL-KC9)?QU=js}Pk2MNG_d;fj8 zp5J26|J63XX8(VS=qR~-QXRP*c<}_RR4%elIS<4!0PDBqt1XhZ3A!w@ZEM0wzp64n z1bmbLJqbxeTtf{Hu+3XR#n^p00?oOKPSdNdE-2m)#FC0C?w7TG?&Cal-!~mR0Eloj z2Z&3U0csIv-@nTJH42ASZ+dks;9I$lgyK8HF&+)q5>^B%c)6P8}Z+ zKX*TAc=GcpdaG`fb*BRh1z+6+;D(=oi@qM>d@O*bK;n*fUFy98NU|30&bvHZTZ-gE zJp_Y`>ViXW^&7KwU)-=I=l8je`ug6=A_%)`Um? zI)3+&+DJB+GLq5^a~xosS$IkfZdIQR(rYXm=9{=@Sk831@V1Tl!X)H_D(#JI_f<*Kd$AGjUAX7>NB<4>ugXQ;bHIe$#l2H0JpuHVc$%CzHjw#ruG zJ1wpI%*t(_U-2qE#!=dq7x${SLTAPH+fBzAs5Mx_ZhHG>PdZ0Wl7FxXxg!Nffg3@# z7i+6X^U+?6=?kFQO)cSZPdp2!>}*4adC$koWhY2MySVk*4iSLt1nRJ-gpO32sW}l* zMm@9J%a_Yeo=+POIJbbe0XRh;$fEtLHuSf&|2^#Nvd?;|d((guLeteUVRJyGpC32p zhS3v_`8PtRMv_Jpa|$QT42u*)c&~8*$5}Icuc(@O%o)nd=cSEx#x4Ee`wzD}2_5l! zbG`>sdB3cezjdkq{;^;8XqS8X-@E$AL6^(QhtLdY&J7D7fACIY;-T8sX&$Bm5Y|*93C@M|S}tmM}k{mfbN1 zi(o!k;rs7c65dfi`Lh<&`I9(HzFOMn~f7@%CTN!(0 z_DmpG0PQdUrFJ|#eY@mfl7$_#1~ud$HWcD+F7YdwRcU9Wlg{eY+*YGE{kZtt<=Un= zA`LCa46j=)XP=Yqxh<*Qci+EHN0LBo4KHxH%09L38px;8Ll@n%Sc|&P6Gex~Cky)p zVR3GrCZx{?>-37pGM_F5LylLb*Fv_WVr*I$*i}7F4Dlo3=#=k@xs+%2`?{2<;ToHZ zoqnDKd-axzTW9$mxFh+pJxx$d9grbTXs9A1yWYC2VywCq?n~HSPE$B-Qz8I-(NK03 zzwwdONXWhF=V}A4V8mg3O?Hr$HWR#AGS={JQq#H>&BI5}Eg{Tw#^!sYjvWGd@Sm<} zweT#LBp5Y-=Q&rf`a9D~sK!xA**HH3E(X?FttVTd*51lq`KU;aIawx4<6gTT`^^Ls zj4dM7jKdGw2a(O*F`tRu$YGeLI^zlUUtquNv(wFi_;*Z#nJ%GPGknVJlBR$` zZ-C#NaS~t>3CuYQwa}7cbpuJuy!y12p7z&|rHPs04@CO=qJF+Hjxy7O zZr0sd${Z|!s#`Wq>aXj=nvIZtR2#`^W217l*7m*#*2;i9qfmG7E`c_BnK&6tXDUdT zu>k7YuR{&E2rsEy4E8ux`IX$08U__Ln7$xgMTTq5h=MP2Ue}>^bcjKjZN?Pp0*D#t zOKENAF5$G`YD0z$bwg5ZE99@8bb`>`#Cs(^8N9PB7mvW56^`7N;wu9z+$c!_B8MvW z!@`=Ls=w%{j*{8d51LnW50~LPr>|Kf@|%%+vb5^Id{zr1jRCj`X5cF_0K~}+R~Q@@ zCasnCV1e+cPyL{&?2qUJ%|aj4rM9zna9&=Xn2I)vmr-Q}4vqU#@iCPROwJ?#PL|jRKWe{E(Lajm0YZ7c*XCNS^DW0Q41V6l=6wo=IJ#w0nJkP%-8Qcy37Gl?K?3#OBqBhmH zaG$g}$*7`tPKl=jIjYxZa$G}vHY|32x|qkU-VcNb#Zb_vv768S4Hoz>nBY&h5QI0Q zjN-3nAo(@1v$2V~q=zXJ^v{9cVx&^3qG20%pcV&5_+ z{TYtlM6in-!twSSYzq5k_Dr`W`*Znf!Np!XlQo=C5z&F`zQGHbBw1|LHN^|wHlY|% z{TElsU2x`Inv2ef6d`?CR}@}pg&M&1kGnvbK2^rkvyg4OM^ZB26RG1HyjAom?^Ulqoy<9^pOMW5a@ zxqdR{d6mcO&i%v4m(7=3_1`GV6#%#iBT+K=SVOs1KGvwm?rZ8hjX*@Une24!C9mW4 z`nxI<5tW9q=(^R5L5=Y`5>%=?8Otk7+qdU#SkipHj2)+Rk)?pDUeZ63jtmp(@?U=s zVbiDkDPVt}SX>WnIxi`NpfsB)9tG_0s&^Vl18sSddyl2qrH<=>pVqC@_Wq(PM~Zdt z5Q;wGt1@l4hKqSA=&dK!7s*)0bT0r?U5K?K#ak~-gpwiHuPYTN7`~A^7oAjkX7f!Y zpj7k;c0Grc)pHXN-n-TH>`T#lZ|rf|LcBOI1a4^?W`#rryyDq}MYr#O)f(*Y&kRXm zjs`@((bpTM@h}a2f>qHw!-~W0Gz&$8deWuIRe7(R!#@=h8b`s5>aq&nA3%IH0C>bG zdic>D0^QIWd-H8!2RjcTFR|p##bFOZKi4@a8LEm0TWu(1wy>-H)bpdTj4Ctj`2*6Q zk;3hEE<%INVN0_f(#bH8MCly0BgJ!q!pqpk-8QnwG-lr8WnNc_9_2;XM{n_5&1L#hKFmf>bsa$$O;8v1=d3)uZrIEJiw*zC~5Q z7(Q9M5neUX;r4T%wizSsI)KiDC;R#i2+Y%%axz_&>HweKfJMp78nbB4Lkbttl%SmR zz{52ZNrhX4=&`8NSh84<6d<{X<)xBqQ|7&-GhxiSBPt?GXtsaWQ7V+5{$pu1 zLHlf1s^=BVXRZ87NP_8I`Zb*#Ib31(hP@UW1Q)iFwuaFfGkP-%$_WL|!pK*}TCwDV zWxJ3&&H@#EVHqd<@+G$y08c3Ie$M|aeErW{@87}YzS9XLoop2su%;wo(^GTDritB2q@P|l$IQS?o?TWLAq8QF5={jJ!(D;*a*n+)1wul5FcYJ z^>%}mB(An_MOcf&7Ka~Sr+6mzP;#?fJgy`j%A7v&m>vzKDaalGL3jMe=-nb~6uDVz z^flr&kD3}=Q!eH#pz8p(bI)6MrEOdjAVb4b)eKLU0)jM=m$WbPvLxVv1Ljlxkc`T zyKE84Xq@ErsHaMUnd*;O?&=j@0s!^sG=%$PGGZ#$xr}MPk?nLxSwd5;>UWnF|8z$~ z2*_BjZ1Qvg)yA}YBJN3qy01PZTCX8AUHr1$YEM1rXYx1a)6Tajx63Zi!&aM-xZ>*b z!b6OtsgMc?aa$myry%k2b1DN%lF$YTkE$>Yu+2~UtdY5pK@lPRQJJIBgVCp`Ewq#w zO2XO+n{blJK>(-sCSv@CShG^mv#OPM|A5s=haWqX#_JP57YkdLvyE*!ceWXT1UFvnN=At!pxy?hoGHgQ@oEC zix~(&oY7UE%7cer&Qr>_TX#T@ubO>ryRp~pAU}FRTEbWN$pXn+y|p^tHg@UswznY7 zcH{dt+J+Yy9009zMJ6pNsu7a z2MozycH@uUXXuCu@`0=#kGAysokk1w&kQBcb&H>Ifw-B~JB^nCJ)rkWt5MJVl2!i= zmDm4yTOcwg1TxzQ?@jU^nj8-Za%HVM@%XX2t1C4CxA{oN|bVw}umpM>#zPzjY8WuZp&yj`@cUA2#O8@MoU8KsDA zY}Ry`+9vTq{DYGGZ^w?3cHYLz09-MQfZb`8spWaP)7*+2rSBTp@wUVK?yjJdWIIy| z2DQ2lv93lX7%qac#ac&;9|;@iCEriF(I3x-Ad)>*)|1HEH|;=X!s{5@Tyf$0No-S9 zhqo0K3+baL85qTcSh^Y=;`UmpyWtPio$!wr2`HN-2Dh!Hh?ky@pvp-?&Uru)Jehm6 zhT0=mKSDkHqT0`+k8_y?R9xz~6s%Y4xP8lTFcvIpOj*gmMETx7N{b8q{2IexR~t2S z%<^i8ExShy$yM-})}qWtk}gfTPdJ$nWE~rH2^ook8XqG(Op zI(eej(NL`;CXs~_OOEjH<;=DbgtDcjgV|eB?Iyc=Mp{2;^|+#xnFi`=Ap1YwKRn+N ztf;{3pDOQXnjqC19Aoay-Nqj@X`DmFtIhD{n?zNCnRx^L(Oc1R=bQ%}D_F^gdR>y1 z^}xZ4m_R->Sb44*X)Zh}rmwe))+=>sH9z3Fz@vai7DzS=nFjj>lP;-A_vVaETB~hF zk~YanP3}-+wa&#O_9TRFXs~6rkLw=pPCwo@E#@s)wSa0PY_HEc|5WEz8p6L^2bte} z-r1;x;$#AK+L_AJw8VvG23wB-%BF7@9or&^%ab)|CLb^K5D}q%+1%X>!TgmYscn63 zRy{0wX2}|})k;Aqpy_4KUxWi;_QZ&vpj+b#;o~PdeQUey1-pr~aZbpUiP5|-i^Z&% zXn+}`=Ud=CoHa^cIQhodV=1OzSGq&P3L%(DCh~IRq_dN;+LfG@}$(H zzOe~+yHz8o?Q)J~4HlkpA$oJq_UyexckMgJXTz^oIv56W*~Tiqpuy#*i%pL@VD?HA zSv&r1l2(|$E7(?(zsaK}#Wm5*8e9v}Jk@Gwfz8JO-@B-^F{?L&4l*<~FF?@c6XP}AMyOC(*N-1X- z$(OIkavgVKj`5fXB(U+Sin`^~kwQJ2p#nzGRSW`A*yI=8Z-?0V8d-nKAk&yvD-jPw zbO7%w6u(Gny3;&FSW$n(icwP&D^7~Jk{%e+9B6ofj}{n{PYR|36#_A`csV$|=SS=o zAuXOud=!en-G76g@ARCIlH);nFn-}>o;x7U%9eBi1eSr)z0YYH*E)MoFMv*H4Jz{d8WFUHqGPb% z+`=COhy}RnT0Mu8%HCvu z`)0p9?6y*_FL-)Pl{_;GVt4P+*3O7k;=5k((8b~+;0FK;M;TMtYeK^5<{KuoHIMcX zYcRXTFO$8&PP~-uD@J)G5FEF-7r+0rRl|Z?*4YN##i1sAxAQXn{G*Rc6sIX~#%!bd z4**>aCk>96L7*=)fvc-Un@w*TzmUY1I(D{x4hEP z*5K2wbdx6bHDqM9rDY5PE6FA{M6GOm7uiH5N{)Z6^c)B)jrp* z{mOipozVQPjk9jxnSKLw)#|+!FAUDh-SY!_y^ZJ@La7luTx+f?xA@I#!e!8m8pcxm zq~NjfC38tPp?+n-(WjYID7%SbkJH^$OyFScFW+C;6qyH zha^d4FL~@(ZFjN&5+?DZC;EB}S0%Of2X8_SmYNgU4a_9gDM04P5n+|euYE*AKC3`bp z)Jbsn==`FGO6)sB?&O6Hj7cVU2K#|%$C`7E(k%26ceW0LQ+;y65YCBXQr;WwXX^e8N5QUyv2Pt(7jO{* z3`*J~*e2h|Yjk@!U?m<)eFX2cBt{Toia4u$xvjG%CMIi(S`d_+gxf=O(hm`*Z--zK zMN5b*hclTVL}flWSt2~iwT9QiGDlIvyag~5>pN@$+Yxs4%u8B+hOVvr)IkQz__?+! zt3}YVys&rJ+_v93;cZ>fV^o_lZ`2)9miG-Z%l(@D)$~e?36{rJWdxQ&5b#KHxE1CjH))Y5?wHz&3})LF3MrAgway z#~k>_1bh(TLNj1qwnNCU2Jh9l4YgTj!ecd#<}<-mEhV!%=s+Uvk`h<1d@R|Md6o Kv6-xR*Z*Iw>s_({ literal 0 HcmV?d00001 diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 7f01b38ddc8..faa02c8f1a9 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -327,6 +327,7 @@ jobs: tests/unit/llms/sagemaker tests/unit/llms/sail tests/unit/llms/sambanova + tests/unit/llms/scaledown tests/unit/llms/sap tests/unit/llms/scaleway tests/unit/llms/searchapi diff --git a/README.md b/README.md index 141e8232de8..9987f13d127 100644 --- a/README.md +++ b/README.md @@ -407,6 +407,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | | | | | | | | | [Sail (`sail`)](https://docs.litellm.ai/docs/providers/sail) | ✅ | ✅ | ✅ | | | | | | | | | [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | | +| [ScaleDown (`scaledown`)](https://docs.litellm.ai/docs/providers/scaledown) | ✅ | | | | | | | | | | | [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | | | [Strands Decider (`strands_decider`)](https://docs.litellm.ai/docs/providers) | | | | | | | | | | | | [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/litellm/__init__.py b/litellm/__init__.py index a6bbd132cbf..554b37f11d4 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -674,6 +674,7 @@ nvidia_nim_models: Set = set() nvidia_riva_models: Set = set() soniox_models: Set = set() sambanova_models: Set = set() +scaledown_models: Final[Set[str]] = set() # Price reloads update this registry through existing references. sambanova_embedding_models: Set = set() novita_models: Set = set() assemblyai_models: Set = set() @@ -756,7 +757,7 @@ def is_openai_finetune_model(key: str) -> bool: return key.startswith("ft:") and not key.count(":") > 1 -def _populate_provider_model_sets(model_cost_map: Dict) -> None: +def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None: for key, value in model_cost_map.items(): if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key): open_ai_chat_completion_models.add(key) @@ -918,6 +919,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None: soniox_models.add(key) elif value.get("litellm_provider") == "sambanova": sambanova_models.add(key) + elif value.get("litellm_provider") == "scaledown": + scaledown_models.add(key) elif value.get("litellm_provider") == "sambanova-embedding-models": sambanova_embedding_models.add(key) elif value.get("litellm_provider") == "novita": @@ -1106,6 +1109,7 @@ model_list = list( | nvidia_riva_models | soniox_models | sambanova_models + | scaledown_models | azure_text_models | novita_models | assemblyai_models @@ -1215,6 +1219,7 @@ def _build_models_by_provider() -> dict: "nvidia_riva": nvidia_riva_models, "soniox": soniox_models, "sambanova": sambanova_models | sambanova_embedding_models, + "scaledown": scaledown_models, "novita": novita_models, "nebius": nebius_models | nebius_embedding_models, "aiml": aiml_models, @@ -2054,6 +2059,9 @@ if TYPE_CHECKING: from .llms.featherless_ai.chat.transformation import ( FeatherlessAIConfig as FeatherlessAIConfig, ) + from .llms.scaledown.chat.transformation import ( + ScaleDownChatConfig as ScaleDownChatConfig, + ) from .llms.cerebras.chat import CerebrasConfig as CerebrasConfig from .llms.nadir.chat.transformation import NadirConfig as NadirConfig from .llms.baseten.chat import BasetenConfig as BasetenConfig diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 9caba6a060a..ce69de3b240 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -277,6 +277,7 @@ LLM_CONFIG_NAMES: Final = ( "NvidiaNimConfig", "NvidiaNimEmbeddingConfig", "FeatherlessAIConfig", + "ScaleDownChatConfig", "CerebrasConfig", "NadirConfig", "BasetenConfig", @@ -1103,6 +1104,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.featherless_ai.chat.transformation", "FeatherlessAIConfig", ), + "ScaleDownChatConfig": ( + ".llms.scaledown.chat.transformation", + "ScaleDownChatConfig", + ), "CerebrasConfig": (".llms.cerebras.chat", "CerebrasConfig"), "NadirConfig": (".llms.nadir.chat.transformation", "NadirConfig"), "BasetenConfig": (".llms.baseten.chat", "BasetenConfig"), diff --git a/litellm/constants.py b/litellm/constants.py index a339fb76976..f9c55f59f42 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -825,6 +825,7 @@ LITELLM_CHAT_PROVIDERS: Final = [ "lemonade", "docker_model_runner", "amazon_nova", + "scaledown", ] # Resolving these providers runs an OAuth device flow (their provider info IS the login), so any diff --git a/litellm/llms/scaledown/__init__.py b/litellm/llms/scaledown/__init__.py new file mode 100644 index 00000000000..d571093869e --- /dev/null +++ b/litellm/llms/scaledown/__init__.py @@ -0,0 +1,3 @@ +from .chat.transformation import ScaleDownChatConfig, ScaleDownError + +__all__ = ["ScaleDownChatConfig", "ScaleDownError"] diff --git a/litellm/llms/scaledown/chat/__init__.py b/litellm/llms/scaledown/chat/__init__.py new file mode 100644 index 00000000000..e6ac554eed1 --- /dev/null +++ b/litellm/llms/scaledown/chat/__init__.py @@ -0,0 +1,3 @@ +from .transformation import ScaleDownChatConfig, ScaleDownError + +__all__ = ["ScaleDownChatConfig", "ScaleDownError"] diff --git a/litellm/llms/scaledown/chat/transformation.py b/litellm/llms/scaledown/chat/transformation.py new file mode 100644 index 00000000000..44eb4785615 --- /dev/null +++ b/litellm/llms/scaledown/chat/transformation.py @@ -0,0 +1,648 @@ +import json +import time +import uuid +from collections.abc import Mapping, Sequence +from functools import reduce +from typing import TYPE_CHECKING, Final, NoReturn +from urllib.parse import unquote + +import httpx +from pydantic import JsonValue, TypeAdapter + +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Choices, Message, ModelResponse, Usage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +JSON_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]]) + +DEFAULT_API_BASE: Final = "https://api.scaledown.xyz" + +DOMAIN_MODELS: Final = frozenset({"classify", "extract", "summarize", "compress"}) + +DECISIONS_MODELS: Final = frozenset({"decisions"}) + +DECISIONS_UPSTREAM_MODEL: Final = "classify-1" + +NATIVE_PATHS: Final[Mapping[str, str]] = { + "classify": "/classify", + "extract": "/extract", + "summarize": "/summarization/abstractive", + "compress": "/compress/raw/", +} + +DECISION_QUESTION_TYPES: Final = frozenset({"choice", "noul", "score"}) + +EXTRA_BODY_KEYS: Final[Mapping[str, frozenset[str]]] = { + "extract": frozenset({"threshold", "top_n"}), + "summarize": frozenset(), + "compress": frozenset({"compression_rate"}), + "classify": frozenset({"labels"}), + "decisions": frozenset({"questions"}), +} + +# Set by the LiteLLM router and proxy on every call; handled by LiteLLM, never sent upstream. +ROUTER_PARAMS: Final = frozenset({"max_retries", "stream_options"}) + +SCORE_MIN_LEVELS: Final = 2 +SCORE_MAX_LEVELS: Final = 10 + +MAX_ENTITIES: Final = 1000 +MAX_REF_CHAIN: Final = 32 +MAX_SCHEMA_DEPTH: Final = 32 + + +class ScaleDownError(BaseLLMException): + pass + + +def _reject(message: str, status_code: int = 400) -> NoReturn: + """The single place this adapter raises its own errors.""" + raise ScaleDownError(status_code=status_code, message=message) + + +class ScaleDownChatConfig(BaseConfig): + @property + def custom_llm_provider(self) -> str | None: + return "scaledown" + + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: BaseConfig signature + model: str, + messages: list[AllMessageValues], # mutable-ok: BaseConfig signature + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + litellm_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, JsonValue]: # mutable-ok: BaseConfig signature + resolved_key: Final = api_key or get_secret_str("SCALEDOWN_API_KEY") + if resolved_key is None: + _reject("Missing ScaleDown API key. Set SCALEDOWN_API_KEY or pass api_key to the call.", 401) + if not api_key and api_base is not None and _root(api_base) != _trusted_root(): + _reject( + "A custom api_base needs its own api_key: the SCALEDOWN_API_KEY from the environment is only " + "sent to the default host or the host in SCALEDOWN_API_BASE." + ) + return {**headers, "x-api-key": resolved_key, "content-type": "application/json"} + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + litellm_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + stream: bool | None = None, + ) -> str: + base: Final = _root(api_base or _trusted_root()) + operation: Final = _operation(model) + if operation in DECISIONS_MODELS: + return f"{base}/v1/scaledown" + return f"{base}{NATIVE_PATHS[operation]}" + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: BaseConfig signature + operation: Final = _operation(model) + if operation == "extract": + return ["response_format", "stream"] + if operation == "summarize": + return ["max_tokens", "max_completion_tokens", "stream"] + return ["stream"] + + def map_openai_params( + self, + non_default_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + model: str, + drop_params: bool, + ) -> dict[str, JsonValue]: # mutable-ok: BaseConfig signature + supported: Final = frozenset(self.get_supported_openai_params(model)) + unsupported: Final = sorted(key for key in non_default_params if key not in supported | ROUTER_PARAMS) + if unsupported and not drop_params: + _reject( + f"ScaleDown model '{model}' does not support {unsupported}. " + f"Supported: {sorted(supported)}. Set litellm.drop_params=True to ignore." + ) + mapped: Final = { + ("max_tokens" if key == "max_completion_tokens" else key): value + for key, value in non_default_params.items() + if key in supported + } + return {**optional_params, **mapped} + + def should_fake_stream( + self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None + ) -> bool: + return bool(stream) + + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: Mapping[str, object], + ) -> Mapping[str, JsonValue]: + # extra_body is merged after guardrails ran on the messages, so it may only add options. + # Anything that carries prompt text or picks the model is refused. + options: Final = JSON_OBJECT.validate_python(extra_body) + operation: Final = _operation(model) + allowed: Final = EXTRA_BODY_KEYS[operation] + unexpected: Final = sorted(set(options) - allowed) + if unexpected: + allowance: Final = f"may only set {sorted(allowed)}" if allowed else "is not accepted" + _reject( + f"extra_body for scaledown/{operation} {allowance}, got {unexpected}. " + "Text and instructions must be passed as messages." + ) + if operation == "compress": + return {"scaledown": {"rate": options["compression_rate"]}} if "compression_rate" in options else {} + if "questions" in options: + _validate_questions(options["questions"]) + if "labels" in options: + _validate_labels(options["labels"]) + return dict(options) + + def sign_request( + self, + headers: dict[str, str], # mutable-ok: BaseConfig signature + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + request_data: dict[str, JsonValue], # mutable-ok: BaseConfig signature + api_base: str, + api_key: str | None = None, + model: str | None = None, + stream: bool | None = None, + fake_stream: bool | None = None, + ) -> tuple[dict[str, str], bytes | None]: # mutable-ok: BaseConfig signature + # Runs after extra_body is merged, so this is where a decisions body is known to be complete. + if model is not None and _operation(model) in DECISIONS_MODELS: + _require_complete_decisions(request_data) + if model is not None and _operation(model) == "classify": + _validate_labels(request_data.get("labels")) + return headers, None + + def transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: BaseConfig signature + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + litellm_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + headers: dict[str, str], # mutable-ok: BaseConfig signature + ) -> dict[str, JsonValue]: # mutable-ok: BaseConfig signature + checked_messages: Final = JSON_MESSAGES.validate_python(messages) + operation: Final = _operation(model) + _validate_text_input(operation, checked_messages, optional_params) + if operation == "classify": + return { + "text": _last_user_text(operation, checked_messages), + **({"labels": optional_params["labels"]} if "labels" in optional_params else {}), + } + if operation in DECISIONS_MODELS: + return dict(_decisions_request(checked_messages, optional_params)) + if operation == "extract": + return dict(_extract_request(checked_messages, optional_params)) + if operation == "summarize": + return dict(_summarize_request(checked_messages, optional_params)) + return dict(_compress_request(checked_messages, optional_params)) + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: "LiteLLMLoggingObj", + request_data: dict[str, JsonValue], # mutable-ok: BaseConfig signature + messages: list[AllMessageValues], # mutable-ok: BaseConfig signature + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + litellm_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + encoding: "Tokenizer | None", + api_key: str | None = None, + json_mode: bool | None = None, + ) -> ModelResponse: + try: + raw: Final = JSON_OBJECT.validate_json(raw_response.content) + except ValueError: + _reject( + f"ScaleDown returned a non-JSON response: {raw_response.text[:500]}", + raw_response.status_code if raw_response.is_error else 502, + ) + + operation: Final = _operation(model) + if operation in DECISIONS_MODELS: + return _decisions_response(operation, raw, model_response) + return _native_response(operation, raw, model_response, request_data.get("entities")) + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, str] | httpx.Headers, # mutable-ok: BaseConfig signature + ) -> BaseLLMException: + return ScaleDownError(status_code=status_code, message=error_message, headers=headers) + + +def _root(api_base: str) -> str: + return api_base.rstrip("/").removesuffix("/v1") + + +def _trusted_root() -> str: + return _root(get_secret_str("SCALEDOWN_API_BASE") or DEFAULT_API_BASE) + + +def _operation(model: str) -> str: + operation: Final = model.split("/", 1)[1] if "/" in model else model + if operation not in DOMAIN_MODELS | DECISIONS_MODELS: + _reject(f"Unknown ScaleDown model '{model}'. Choose from {sorted(DOMAIN_MODELS | DECISIONS_MODELS)}.") + return operation + + +def _text_of(message: Mapping[str, JsonValue]) -> str | None: + content: Final = message.get("content") + if isinstance(content, str): + return content + if isinstance(content, list): + parts: Final = [ + str(part.get("text", "")) for part in content if isinstance(part, dict) and part.get("type") == "text" + ] + return "\n".join(parts) if parts else None + return None + + +def _validate_text_input( + operation: str, messages: Sequence[Mapping[str, JsonValue]], optional_params: Mapping[str, JsonValue] +) -> None: + if any(key in optional_params for key in ("state", "document", "document_mime_type", "text")): + _reject("ScaleDown accepts text through messages only. Native state and document inputs are not supported.") + for message in messages: + if operation in {"classify", "decisions", "extract"} and message.get("role") in {"system", "developer"}: + _reject( + f"ScaleDown {operation} does not support system messages. Put instructions in the field definitions." + ) + if _has_non_text_content(message): + _reject("ScaleDown currently supports text only. Image and document inputs are not supported.") + + +def _has_non_text_content(message: Mapping[str, JsonValue]) -> bool: + content: Final = message.get("content") + return isinstance(content, list) and any( + not isinstance(part, dict) or part.get("type") != "text" for part in content + ) + + +def _last_user_text(operation: str, messages: Sequence[Mapping[str, JsonValue]]) -> str: + message: Final = next((message for message in reversed(messages) if message.get("role") == "user"), None) + text: Final = _text_of(message) if message is not None else None + if not text: + _reject(f"ScaleDown {operation} requires text in the last user message.") + return text + + +def _validate_labels(labels: JsonValue) -> None: + if not isinstance(labels, list) or not labels: + _reject("ScaleDown classify requires a non-empty 'labels' list of objects with 'name' and 'rubric' strings.") + for label in labels: + if not isinstance(label, Mapping) or not all( + isinstance(label.get(key), str) and label[key] for key in ("name", "rubric") + ): + _reject("Each classify label needs non-empty 'name' and 'rubric' strings.") + + +def _child(node: JsonValue, key: str) -> JsonValue: + return node.get(key.replace("~1", "/").replace("~0", "~")) if isinstance(node, Mapping) else None + + +def _extend_ref_path(path: tuple[str, ...], ref: str) -> tuple[str, ...]: + return (*path, ref) + + +def _nullable_branch(schema: Mapping[str, JsonValue]) -> Mapping[str, JsonValue] | None: + if "allOf" in schema: + _reject("ScaleDown extraction does not support allOf. Supply the object's properties directly.") + variants: Final = schema.get("anyOf", schema.get("oneOf")) + if variants is None: + return None + if any(key in schema for key in ("properties", "items", "$ref", "type")) or ( + "anyOf" in schema and "oneOf" in schema + ): + _reject("ScaleDown extraction cannot combine nullable branches with sibling schema definitions.") + if not isinstance(variants, list) or len(variants) != 2: + _reject("ScaleDown extraction supports anyOf/oneOf only for one schema plus null.") + branches: Final = tuple(branch for branch in variants if isinstance(branch, dict)) + non_null: Final = tuple(branch for branch in branches if branch.get("type") != "null") + if len(branches) != 2 or len(non_null) != 1: + _reject("ScaleDown extraction supports anyOf/oneOf only for one schema plus null.") + return { + **non_null[0], + **{key: value for key, value in schema.items() if key not in {"anyOf", "oneOf"}}, + } + + +def _deref( + prop: Mapping[str, JsonValue], root: Mapping[str, JsonValue], path: tuple[str, ...] +) -> tuple[Mapping[str, JsonValue], tuple[str, ...], bool]: + """Follow nullable branches and local `$ref`s (`#/$defs/X`, `#/definitions/X`). + + Returns the schema, the refs followed so far, and whether a ref repeated (a cycle). + The walk is a bounded loop, so a long chain of distinct refs cannot exhaust the stack. + """ + current: Mapping[str, JsonValue] = prop # rebind-ok: bounded walk along a ref chain + followed: tuple[str, ...] = path # rebind-ok: bounded walk along a ref chain + for _ in range(MAX_REF_CHAIN): + nullable: Mapping[str, JsonValue] | None = _nullable_branch(current) + ref: JsonValue = current.get("$ref") + if nullable is not None: + current = nullable + continue + if "$ref" not in current: + return current, followed, False + if not isinstance(ref, str) or not ref.startswith("#"): + _reject("ScaleDown extraction supports only local JSON Pointer $refs.") + pointer: str = unquote(ref[1:]) + if not pointer.startswith("/"): + _reject("ScaleDown extraction supports only local JSON Pointer $refs beginning with #/.") + if any(key in current for key in ("properties", "items", "type")): + _reject("ScaleDown extraction cannot combine $ref with sibling schema definitions.") + if ref in followed: + return current, followed, True + target: JsonValue = reduce(_child, pointer[1:].split("/"), dict(root)) + if not isinstance(target, Mapping): + _reject(f"ScaleDown extraction cannot resolve $ref '{ref}' to a schema object.") + current, followed = target, _extend_ref_path(followed, ref) + _reject(f"The response_format schema follows more than {MAX_REF_CHAIN} chained $refs or nullable branches.") + + +def _build_properties( + properties: Mapping[str, JsonValue], + root: Mapping[str, JsonValue], + path: tuple[str, ...], + budget: int, + depth: int = 0, +) -> tuple[dict[str, JsonValue], int]: # mutable-ok: serialized JSON entity map + """Turn JSON-schema properties into the /extract entity map, spending one budget unit per entity. + + Each property becomes an entity whose value is its description (or its name when there is + none). Nested objects stay nested, arrays of objects become a one-element list holding the + item's entity map, local `$ref`s are followed, and a ref that points back into itself is cut + off at that point instead of expanded. + """ + if depth > MAX_SCHEMA_DEPTH: + _reject(f"The response_format schema nests deeper than {MAX_SCHEMA_DEPTH} levels.") + built: Final[dict[str, JsonValue]] = {} # mutable-ok: filled once while threading the budget through + remaining = budget # rebind-ok: the budget is threaded through the loop + for name, prop in properties.items(): + if remaining <= 0: + _reject( + f"The response_format schema expands to more than {MAX_ENTITIES} entities; " + "remove recursive or heavily repeated $refs." + ) + built[name], remaining = _build_entity(name, prop, root, path, remaining - 1, depth) + return built, remaining + + +def _build_entity( + name: str, prop: JsonValue, root: Mapping[str, JsonValue], path: tuple[str, ...], budget: int, depth: int +) -> tuple[JsonValue, int]: + schema, followed, cyclic = _deref(prop if isinstance(prop, Mapping) else {}, root, path) + leaf: Final = schema.get("description") or name + nested: Final = schema.get("properties") + if cyclic: + return leaf, budget + if isinstance(nested, Mapping): + return _build_properties(nested, root, followed, budget, depth + 1) + raw_items: Final = schema.get("items") + schema_type: Final = schema.get("type") + is_array: Final = schema_type == "array" or (isinstance(schema_type, list) and "array" in schema_type) + if is_array and isinstance(raw_items, Mapping): + items, item_path, item_cyclic = _deref(raw_items, root, followed) + item_properties: Final = items.get("properties") + if not item_cyclic and isinstance(item_properties, Mapping): + built, remaining = _build_properties(item_properties, root, item_path, budget, depth + 1) + return [built], remaining + return leaf, budget + + +def _extract_request( + messages: Sequence[Mapping[str, JsonValue]], optional_params: Mapping[str, JsonValue] +) -> Mapping[str, JsonValue]: + response_format: Final = optional_params.get("response_format") + json_schema: Final = response_format.get("json_schema") if isinstance(response_format, Mapping) else None + if isinstance(json_schema, Mapping) and json_schema.get("strict") is True: + _reject( + "ScaleDown extract does not support strict JSON schemas. Use strict=false and validate the returned fields." + ) + schema: Final = json_schema.get("schema") if isinstance(json_schema, Mapping) else None + properties: Final = _deref(schema, schema, ())[0].get("properties") if isinstance(schema, Mapping) else None + if not isinstance(schema, dict) or not isinstance(properties, dict) or not properties: + _reject( + "ScaleDown extract needs response_format={'type': 'json_schema', ...} with a non-empty " + "'properties' map; each property becomes an entity and its description the extraction hint." + ) + return { + "text": _last_user_text("extract", messages), + "entities": _build_properties(properties, schema, (), MAX_ENTITIES)[0], + **{key: optional_params[key] for key in ("threshold", "top_n") if key in optional_params}, + } + + +def _summarize_request( + messages: Sequence[Mapping[str, JsonValue]], optional_params: Mapping[str, JsonValue] +) -> Mapping[str, JsonValue]: + instructions: Final = [ + text for message in messages if message.get("role") in {"system", "developer"} and (text := _text_of(message)) + ] + return { + "text": _last_user_text("summarize", messages), + **({"instructions": "\n".join(instructions)} if instructions else {}), + **({"max_tokens": optional_params["max_tokens"]} if "max_tokens" in optional_params else {}), + } + + +def _compress_request( + messages: Sequence[Mapping[str, JsonValue]], optional_params: Mapping[str, JsonValue] +) -> Mapping[str, JsonValue]: + prompt: Final = _last_user_text("compress", messages) + last_user_index: Final = max(i for i, m in enumerate(messages) if m.get("role") == "user" and _text_of(m)) + context: Final = "\n\n".join(text for i, m in enumerate(messages) if i != last_user_index and (text := _text_of(m))) + return { + "context": context, + "prompt": prompt, + "scaledown": {"rate": optional_params.get("compression_rate", "auto")}, + } + + +def _decisions_request( + messages: Sequence[Mapping[str, JsonValue]], optional_params: Mapping[str, JsonValue] +) -> Mapping[str, JsonValue]: + questions: Final = optional_params.get("questions") + if questions is not None: + _validate_questions(questions) + return { + "model": DECISIONS_UPSTREAM_MODEL, + "state": {"text": _last_user_text("decisions", messages)}, + **({"questions": questions} if questions is not None else {}), + } + + +def _require_complete_decisions(body: Mapping[str, JsonValue]) -> None: + if not body.get("questions"): + _reject( + "ScaleDown decisions requires a non-empty 'questions' map, passed as an extra parameter, " + 'e.g. {"questions": {"category": {"type": "choice", "criteria": {"billing": "..."}}}}. ' + "Question types are 'choice', 'noul', and 'score'." + ) + + +def _validate_questions(questions: JsonValue) -> None: + if not isinstance(questions, Mapping): + _reject(f"'questions' must be a map of name to question, got {type(questions).__name__}.") + for name, question in questions.items(): + if not isinstance(question, Mapping): + _reject(f"Question '{name}' must be an object.") + question_type = question.get("type") + if not isinstance(question_type, str) or question_type not in DECISION_QUESTION_TYPES: + _reject(f"Question '{name}' has type {question_type!r}; expected one of {sorted(DECISION_QUESTION_TYPES)}.") + criteria = question.get("criteria") + if question_type == "choice" and (not isinstance(criteria, Mapping) or not criteria): + _reject(f"Choice question '{name}' needs a non-empty 'criteria' map of option key to description.") + if question_type == "score" and ( + not isinstance(criteria, list) or not SCORE_MIN_LEVELS <= len(criteria) <= SCORE_MAX_LEVELS + ): + _reject( + f"Score question '{name}' needs 'criteria' as an ordered list of " + f"{SCORE_MIN_LEVELS} to {SCORE_MAX_LEVELS} level descriptions, lowest to highest." + ) + + +def _unwrap(value: JsonValue) -> JsonValue: + wrapper_keys: Final = {"_value", "_span_anchor"} + if isinstance(value, Mapping) and "_value" in value and set(value) <= wrapper_keys: + return value["_value"] + return value + + +def _clean_extraction(value: JsonValue, requested: JsonValue) -> JsonValue: + """Reduce an extraction result to the fields the caller's schema asked for. + + The request's entity map says which keys are real fields, so provider-added keys such as + `_span_anchor` and `{"_value": ..., "_span_anchor": ...}` wrappers are dropped without + touching a requested field that happens to share one of those names. The untouched payload + stays on `_hidden_params["scaledown_response"]`. + """ + if isinstance(requested, Mapping) and isinstance(value, Mapping): + return {key: _clean_extraction(value[key], requested[key]) for key in requested if key in value} + if isinstance(requested, list) and requested and isinstance(value, list): + return [_clean_extraction(item, requested[0]) for item in value] + return _unwrap(value) + + +def _extracted_fields(raw: Mapping[str, JsonValue], requested: JsonValue) -> JsonValue: + entities: Final = raw.get("entities") + matches: Final = ( + tuple(entity for entity in entities if isinstance(entity, Mapping)) if isinstance(entities, list) else () + ) + scalar_names: Final = ( + tuple(name for name, hint in requested.items() if isinstance(hint, str)) + if isinstance(requested, Mapping) + else () + ) + scalars: Final = { + name: next(entity["text"] for entity in matches if entity.get("type") == name and "text" in entity) + for name in scalar_names + if any(entity.get("type") == name and "text" in entity for entity in matches) + } + structured: Final = raw.get("structured_result") + return _clean_extraction({**scalars, **(structured if isinstance(structured, Mapping) else {})}, requested) + + +def _native_response( + operation: str, raw: JsonValue, model_response: ModelResponse, requested: JsonValue = None +) -> ModelResponse: + """Wrap a native /extract, /summarization/abstractive or /compress/raw/ payload. + + Summarize and compress return the upstream payload as JSON on choices[0].message.content. + Extract returns the fields requested by response_format without enforcing schema types. + The native APIs report input tokens only; they return no output token + count, so completion_tokens stays 0 because it is unmeasured, not because nothing was generated. + """ + if not isinstance(raw, Mapping) or not raw: + _reject( + f"ScaleDown '{operation}' returned an empty or malformed response: {json.dumps(raw)[:500]}", + 500, + ) + + results: Final = raw.get("results") + input_tokens: Final = raw.get("input_tokens") or ( + (results.get("original_prompt_tokens") if isinstance(results, Mapping) else None) + or raw.get("original_prompt_tokens") + if operation == "compress" + else None + ) + content: Final = _extracted_fields(raw, requested) if operation == "extract" else raw + + model_response.id = f"chatcmpl-{uuid.uuid4().hex}" # rebind-ok: LiteLLM fills the response object it passes in + model_response.created = int(time.time()) # rebind-ok: LiteLLM fills the response object it passes in + model_response.model = f"scaledown/{operation}" # rebind-ok: LiteLLM fills the response object it passes in + model_response.object = "chat.completion" # rebind-ok: LiteLLM fills the response object it passes in + model_response.choices = [ # rebind-ok: LiteLLM fills the response object it passes in + Choices( + index=0, + message=Message(role="assistant", content=json.dumps(content, separators=(",", ":"))), + finish_reason="stop", + ) + ] + _set_usage(model_response, {"prompt_tokens": input_tokens or 0}) + model_response.hidden_params["scaledown_response"] = raw # pyright: ignore[reportUnknownMemberType] # Inherited hidden_params has untyped values. # rebind-ok: supplied response + return model_response + + +def _decisions_response(operation: str, raw: JsonValue, model_response: ModelResponse) -> ModelResponse: + answers: Final = raw.get("answers") if isinstance(raw, Mapping) else None + if answers is None: + _reject(f"ScaleDown decisions response contained no answers: {json.dumps(raw)[:500]}", 500) + + model_response.id = f"chatcmpl-{uuid.uuid4().hex}" # rebind-ok: LiteLLM fills the response object it passes in + model_response.created = int(time.time()) # rebind-ok: LiteLLM fills the response object it passes in + model_response.model = f"scaledown/{operation}" # rebind-ok: LiteLLM fills the response object it passes in + model_response.object = "chat.completion" # rebind-ok: LiteLLM fills the response object it passes in + model_response.choices = [ # rebind-ok: LiteLLM fills the response object it passes in + Choices( + index=0, + message=Message(role="assistant", content=json.dumps(answers, separators=(",", ":"))), + finish_reason="stop", + ) + ] + + usage: Final = raw.get("usage") if isinstance(raw, dict) else None + counts: Final = usage if isinstance(usage, dict) else {} + _set_usage( + model_response, + { + "prompt_tokens": counts.get("input_tokens", 0), + "completion_tokens": counts.get("output_tokens", 0), + }, + ) + model_response.hidden_params["scaledown_response"] = raw # pyright: ignore[reportUnknownMemberType] # Inherited hidden_params has untyped values. # rebind-ok: supplied response + return model_response + + +def _set_usage(model_response: ModelResponse, usage: Mapping[str, JsonValue]) -> None: + prompt_tokens: Final = _token_count(usage.get("prompt_tokens")) + completion_tokens: Final = _token_count(usage.get("completion_tokens")) + total_tokens: Final = usage.get("total_tokens") + model_response.usage = Usage( # pyright: ignore[reportAttributeAccessIssue] # ModelResponse stores usage as a dynamic Pydantic field. # rebind-ok: fills the supplied response + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=_token_count(total_tokens) if total_tokens is not None else prompt_tokens + completion_tokens, + ) + + +def _token_count(value: JsonValue) -> int: + if value is None: + return 0 + if isinstance(value, int) and value >= 0: + return value + _reject("ScaleDown returned an invalid token count.", 500) diff --git a/litellm/main.py b/litellm/main.py index 88c1257ee2c..c2c6d5affe8 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1938,6 +1938,39 @@ def _complete_ragflow(ctx: CompletionDispatchContext) -> _CompletionDispatchResu return response +def _complete_scaledown(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: + try: + return base_llm_http_handler.completion( + model=ctx.model, + messages=ctx.messages, + headers=ctx.headers, + model_response=ctx.model_response, + api_key=ctx.api_key, + api_base=ctx.api_base, + acompletion=ctx.acompletion, + logging_obj=ctx.logging, + optional_params=ctx.optional_params, + litellm_params=ctx.litellm_params, + shared_session=ctx.shared_session, + timeout=cast( # cast-ok: CompletionTimeout.resolve normalizes this value + float | httpx.Timeout, ctx.timeout + ), + client=_dispatch_client_http(ctx), + custom_llm_provider=ctx.custom_llm_provider, + encoding=_get_encoding(), + stream=ctx.stream, + provider_config=ctx.provider_config, + ) + except Exception as exc: + ctx.logging.post_call( + input=ctx.messages, + api_key=ctx.api_key, + original_response=str(exc), + additional_args={"headers": ctx.headers}, + ) + raise + + def _complete_xai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: acompletion: Final = ctx.acompletion api_base: Final = ctx.api_base @@ -5859,6 +5892,8 @@ def completion( elif custom_llm_provider == "ragflow": ## COMPLETION CALL - RAGFlow uses HTTP handler to support custom URL paths response = _complete_ragflow(_dispatch_ctx) + elif custom_llm_provider == "scaledown": + response = _complete_scaledown(_dispatch_ctx) # rebind-ok: shared provider return elif custom_llm_provider == "xai": ## COMPLETION CALL response = _complete_xai(_dispatch_ctx) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cb910ae12ea..20558be3422 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -71842,5 +71842,51 @@ "image" ], "supports_embedding_image_input": true + }, + "scaledown/extract": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/summarize": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": true, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/compress": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": true, + "supports_function_calling": false, + "supports_vision": false + }, + "scaledown/classify": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/decisions": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false } } diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index d9a6feb625d..a0e75b1f02d 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -2064,6 +2064,24 @@ "search": true } }, + "scaledown": { + "display_name": "ScaleDown (`scaledown`)", + "url": "https://docs.litellm.ai/docs/providers/scaledown", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "sambanova": { "display_name": "Sambanova (`sambanova`)", "url": "https://docs.litellm.ai/docs/providers/sambanova", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index f20369e279a..633e9edf71c 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -1,4 +1,22 @@ [ + { + "provider": "SCALEDOWN", + "provider_display_name": "ScaleDown", + "litellm_provider": "scaledown", + "credential_fields": [ + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "scaledown/summarize" + }, { "provider": "AIML", "provider_display_name": "AI/ML API", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d6751610e7a..13fd86b74a7 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4224,6 +4224,7 @@ class LlmProviders(str, Enum): FRIENDLIAI = "friendliai" FEATHERLESS_AI = "featherless_ai" WATSONX = "watsonx" + SCALEDOWN = "scaledown" WATSONX_TEXT = "watsonx_text" TRITON = "triton" PREDIBASE = "predibase" diff --git a/litellm/utils.py b/litellm/utils.py index e7a171f56c6..6774771dbf9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8655,6 +8655,7 @@ class ProviderConfigManager: False, ), LlmProviders.FEATHERLESS_AI: (lambda: litellm.FeatherlessAIConfig(), False), + LlmProviders.SCALEDOWN: (lambda: litellm.ScaleDownChatConfig(), False), LlmProviders.NOVITA: (lambda: litellm.NovitaConfig(), False), LlmProviders.NEBIUS: (lambda: litellm.NebiusConfig(), False), LlmProviders.WANDB: (lambda: litellm.WandbConfig(), False), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cb910ae12ea..20558be3422 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -71842,5 +71842,51 @@ "image" ], "supports_embedding_image_input": true + }, + "scaledown/extract": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/summarize": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": true, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/compress": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": true, + "supports_function_calling": false, + "supports_vision": false + }, + "scaledown/classify": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/decisions": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false } } diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 33ac4e4e60b..ea617cf8462 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2349,6 +2349,24 @@ "search": true } }, + "scaledown": { + "display_name": "ScaleDown (`scaledown`)", + "url": "https://docs.litellm.ai/docs/providers/scaledown", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "sambanova": { "display_name": "Sambanova (`sambanova`)", "url": "https://docs.litellm.ai/docs/providers/sambanova", diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 7166891bb1a..cea2b4154a7 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -67,6 +67,7 @@ IGNORE_FUNCTIONS = [ "strict_json_schema", # harness: max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by raising ValueError at the cap. "toml_value", # harness/codex: max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by raising OptionsMismatch at the cap. "with_json_string_leaves", # transitively bounded: only runs on a tree json_string_leaves already walked under the cap. + "_clean_extraction", # ScaleDown: traverses the entity map built under MAX_SCHEMA_DEPTH=32 and MAX_ENTITIES=1000. "json_unrewritable_labels", # max depth set (MAX_STRUCTURED_CONTENT_SCAN_DEPTH); returns the None sentinel at the cap so the caller blocks. "_flatten_form_field", # bounded by the nesting depth of the already-parsed request body (a finite JSON tree, no cycles possible). "_flatten_form_data_field", # bounded by the nesting depth of the already-parsed request body (a finite JSON tree, no cycles possible). diff --git a/tests/unit/llms/scaledown/__init__.py b/tests/unit/llms/scaledown/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/scaledown/chat/__init__.py b/tests/unit/llms/scaledown/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/scaledown/chat/test_scaledown_chat_transformation.py b/tests/unit/llms/scaledown/chat/test_scaledown_chat_transformation.py new file mode 100644 index 00000000000..82f03a2cd80 --- /dev/null +++ b/tests/unit/llms/scaledown/chat/test_scaledown_chat_transformation.py @@ -0,0 +1,1213 @@ +import json +from collections.abc import Mapping +from queue import SimpleQueue +from typing import Final, Literal + +import httpx +import pytest +import respx +from pydantic import JsonValue + +import litellm +from litellm.llms.scaledown.chat.transformation import ( + DECISIONS_UPSTREAM_MODEL, + ScaleDownChatConfig, + ScaleDownError, +) +from litellm.types.utils import ModelResponse + +BASE = "https://api.scaledown.xyz" + + +@pytest.fixture +def config() -> ScaleDownChatConfig: + return ScaleDownChatConfig() + + +@pytest.fixture(autouse=True) +def scaledown_api_key(monkeypatch): + monkeypatch.setenv("SCALEDOWN_API_KEY", "sk-scaledown-test") + monkeypatch.delenv("SCALEDOWN_API_BASE", raising=False) + + +def _transform_response( + config: ScaleDownChatConfig, model: str, payload: dict, request_data: dict | None = None +) -> ModelResponse: + return config.transform_response( + model=model, + raw_response=httpx.Response( + status_code=200, + json=payload, + request=httpx.Request("POST", f"{BASE}/extract"), + ), + model_response=ModelResponse(), + logging_obj=None, + request_data=request_data or {}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + +def _build_decisions(config: ScaleDownChatConfig, messages: list, optional_params: dict) -> dict: + body = config.transform_request( + model="scaledown/decisions", messages=messages, optional_params=optional_params, litellm_params={}, headers={} + ) + config.sign_request( + headers={}, + optional_params=optional_params, + request_data=body, + api_base=f"{BASE}/v1/scaledown", + model="scaledown/decisions", + ) + return body + + +def test_auth_uses_x_api_key_not_bearer(config): + headers = config.validate_environment( + headers={}, + model="scaledown/extract", + messages=[], + optional_params={}, + litellm_params={}, + api_key="sk-explicit", + ) + + assert headers["x-api-key"] == "sk-explicit" + assert "Authorization" not in headers + + +def test_missing_api_key_is_rejected_before_any_request(config, monkeypatch): + monkeypatch.delenv("SCALEDOWN_API_KEY", raising=False) + + with pytest.raises(ScaleDownError) as exc: + config.validate_environment( + headers={}, + model="scaledown/extract", + messages=[], + optional_params={}, + litellm_params={}, + ) + + assert exc.value.status_code == 401 + + +@pytest.mark.parametrize( + "model, path", + [ + ("scaledown/extract", "/extract"), + ("scaledown/summarize", "/summarization/abstractive"), + ("scaledown/compress", "/compress/raw/"), + ("scaledown/classify", "/classify"), + ("scaledown/decisions", "/v1/scaledown"), + ], +) +def test_each_model_targets_its_native_endpoint(config, model, path): + url = config.get_complete_url(api_base=None, api_key="k", model=model, optional_params={}, litellm_params={}) + + assert url == f"{BASE}{path}" + + +@pytest.mark.parametrize( + "api_base", + ["https://staging.scaledown.xyz", "https://staging.scaledown.xyz/", "https://staging.scaledown.xyz/v1"], +) +def test_api_base_with_or_without_v1_resolves_to_the_same_host(config, api_base): + extract = config.get_complete_url( + api_base=api_base, api_key="k", model="scaledown/extract", optional_params={}, litellm_params={} + ) + decisions = config.get_complete_url( + api_base=api_base, api_key="k", model="scaledown/decisions", optional_params={}, litellm_params={} + ) + + assert extract == "https://staging.scaledown.xyz/extract" + assert decisions == "https://staging.scaledown.xyz/v1/scaledown" + + +def test_decisions_request_carries_state_and_questions_without_chat_keys(config): + questions = { + "category": { + "type": "choice", + "instructions": "Which category?", + "criteria": {"billing": "A charge or refund.", "technical": "A bug."}, + } + } + + body = config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "I was charged twice."}], + optional_params={"questions": questions}, + litellm_params={}, + headers={}, + ) + + assert body == { + "model": DECISIONS_UPSTREAM_MODEL, + "state": {"text": "I was charged twice."}, + "questions": questions, + } + + +def test_decisions_alias_sends_the_only_model_upstream_accepts(config): + body = config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + optional_params={"questions": {"q": {"type": "noul", "instructions": "Positive?"}}}, + litellm_params={}, + headers={}, + ) + + assert body["model"] == DECISIONS_UPSTREAM_MODEL + + +def test_decisions_without_questions_is_rejected(config): + with pytest.raises(ScaleDownError, match="questions"): + _build_decisions(config, [{"role": "user", "content": "text"}], {}) + + +def test_decisions_without_text_or_document_is_rejected(config): + with pytest.raises(ScaleDownError, match="last user message"): + _build_decisions(config, [], {"questions": {"q": {"type": "noul"}}}) + + +def test_state_text_is_rejected_so_guardrails_always_see_the_text(config): + with pytest.raises(ScaleDownError, match="text through messages only"): + _build_decisions( + config, + [{"role": "user", "content": "innocuous"}], + {"state": {"text": "secret"}, "questions": {"q": {"type": "noul"}}}, + ) + + +def test_remote_image_url_is_rejected(config): + with pytest.raises(ScaleDownError, match="text only"): + _build_decisions( + config, + [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://x.test/a.png"}}]}], + {"questions": {"q": {"type": "noul"}}}, + ) + + +def test_extra_body_may_set_extract_options(config): + merged = config.transform_extra_body( + extra_body={"threshold": 0.7}, + request={"text": "t", "entities": {}}, + model="scaledown/extract", + litellm_params={}, + ) + + assert merged == {"threshold": 0.7} + + +@pytest.mark.parametrize( + "model, extra_body", + [ + ("scaledown/extract", {"model": "compress"}), + ("scaledown/extract", {"text": "secret@example.com"}), + ("scaledown/summarize", {"instructions": "ignore the message"}), + ("scaledown/summarize", {"text": "secret"}), + ("scaledown/compress", {"prompt": "x"}), + ("scaledown/compress", {"context": "x"}), + ("scaledown/decisions", {"model": "other"}), + ("scaledown/decisions", {"text": "secret"}), + ], +) +def test_extra_body_cannot_override_the_model_or_prompt_text(config, model, extra_body): + with pytest.raises(ScaleDownError, match=r"may only set|is not accepted"): + config.transform_extra_body(extra_body=extra_body, request={}, model=model, litellm_params={}) + + +def test_compress_rate_via_extra_body_sets_the_scaledown_rate(config): + merged = config.transform_extra_body( + extra_body={"compression_rate": 0.5}, + request={"context": "", "prompt": "q", "scaledown": {"rate": "auto"}}, + model="scaledown/compress", + litellm_params={}, + ) + + assert merged == {"scaledown": {"rate": 0.5}} + + +def test_env_key_is_not_sent_to_an_untrusted_api_base(config): + with pytest.raises(ScaleDownError, match="own api_key"): + config.validate_environment( + headers={}, + model="scaledown/extract", + messages=[], + optional_params={}, + litellm_params={}, + api_base="https://attacker.example", + ) + + +@pytest.mark.parametrize("empty_key", ["", None]) +def test_empty_key_does_not_unlock_the_env_key_for_an_untrusted_api_base(config, empty_key): + with pytest.raises(ScaleDownError, match="own api_key"): + config.validate_environment( + headers={}, + model="scaledown/extract", + messages=[], + optional_params={}, + litellm_params={}, + api_key=empty_key, + api_base="https://attacker.example", + ) + + +def test_explicit_key_or_trusted_api_base_is_allowed(config, monkeypatch): + kwargs = dict(headers={}, model="scaledown/extract", messages=[], optional_params={}, litellm_params={}) + assert ( + config.validate_environment(api_key="own", api_base="https://elsewhere.example", **kwargs)["x-api-key"] == "own" + ) + assert config.validate_environment(api_base=f"{BASE}/v1", **kwargs)["x-api-key"] == "sk-scaledown-test" + monkeypatch.setenv("SCALEDOWN_API_BASE", "https://staging.scaledown.xyz") + assert ( + config.validate_environment(api_base="https://staging.scaledown.xyz/", **kwargs)["x-api-key"] + == "sk-scaledown-test" + ) + + +def test_max_completion_tokens_maps_to_max_tokens(config): + assert config.map_openai_params( + non_default_params={"max_completion_tokens": 50}, + optional_params={}, + model="scaledown/summarize", + drop_params=False, + ) == {"max_tokens": 50} + + +def test_router_params_are_accepted_and_not_forwarded(config): + assert ( + config.map_openai_params( + non_default_params={"max_retries": 2}, optional_params={}, model="scaledown/decisions", drop_params=False + ) + == {} + ) + + +def test_every_model_fakes_a_stream(config): + assert config.should_fake_stream(model="scaledown/compress", stream=True) + assert not config.should_fake_stream(model="scaledown/compress", stream=False) + + +def test_unknown_question_type_is_rejected(config): + with pytest.raises(ScaleDownError, match="expected one of"): + config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "t"}], + optional_params={"questions": {"q": {"type": "ranking"}}}, + litellm_params={}, + headers={}, + ) + + +def test_choice_question_without_criteria_is_rejected(config): + with pytest.raises(ScaleDownError, match="non-empty 'criteria' map"): + config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "t"}], + optional_params={"questions": {"q": {"type": "choice", "criteria": {}}}}, + litellm_params={}, + headers={}, + ) + + +@pytest.mark.parametrize("levels", [1, 11]) +def test_score_criteria_outside_two_to_ten_is_rejected(config, levels): + with pytest.raises(ScaleDownError, match="ordered list"): + config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "t"}], + optional_params={"questions": {"q": {"type": "score", "criteria": ["l"] * levels}}}, + litellm_params={}, + headers={}, + ) + + +@pytest.mark.parametrize("levels", [2, 10]) +def test_score_criteria_within_two_to_ten_is_accepted(config, levels): + criteria = [f"level {index}" for index in range(levels)] + + body = config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "t"}], + optional_params={"questions": {"q": {"type": "score", "criteria": criteria}}}, + litellm_params={}, + headers={}, + ) + + assert body["questions"]["q"]["criteria"] == criteria + + +def test_extract_content_is_the_clean_fields_and_raw_payload_is_kept(config): + payload = { + "entities": [], + "structured_result": { + "invoice": { + "vendor": "Northwind", + "vendor_span_anchor": "Invoice from Northwind", + "customer": {"_value": "Ada Lovelace", "_span_anchor": "to Ada Lovelace"}, + "amount": 500, + } + }, + "input_tokens": 171, + } + + requested = {"invoice": {"vendor": "v", "customer": "c", "amount": "a"}} + response = _transform_response(config, "scaledown/extract", payload, {"entities": requested}) + + assert json.loads(response.choices[0].message.content) == { + "invoice": {"vendor": "Northwind", "customer": "Ada Lovelace", "amount": 500} + } + assert response.model == "scaledown/extract" + assert response.usage.prompt_tokens == 171 + assert response.usage.completion_tokens == 0 + assert response._hidden_params["scaledown_response"] == payload + + +def test_extract_cleaning_keeps_requested_fields_named_like_provider_keys(config): + payload = {"structured_result": {"_value": "x", "vendor": "y", "note_span_anchor": "z", "note": "n"}} + requested = {"_value": "v", "vendor": "v", "note_span_anchor": "s", "note": "n"} + + response = _transform_response(config, "scaledown/extract", payload, {"entities": requested}) + + assert json.loads(response.choices[0].message.content) == payload["structured_result"] + + +def test_extract_cleaning_handles_lists_of_objects(config): + payload = {"structured_result": {"items": [{"sku": "a", "sku_span_anchor": "s"}, {"sku": {"_value": "b"}}]}} + + response = _transform_response(config, "scaledown/extract", payload, {"entities": {"items": [{"sku": "code"}]}}) + + assert json.loads(response.choices[0].message.content) == {"items": [{"sku": "a"}, {"sku": "b"}]} + + +def test_long_ref_chain_is_rejected_cleanly_not_with_a_recursion_error(config): + defs = {f"D{i}": {"$ref": f"#/$defs/D{i + 1}"} for i in range(5000)} + defs["D5000"] = {"type": "string"} + schema = {"type": "object", "$defs": defs, "properties": {"f": {"$ref": "#/$defs/D0"}}} + + with pytest.raises(ScaleDownError, match="chained"): + _extract_body(config, schema) + + +def test_deeply_nested_schema_is_rejected_cleanly(config): + nested: dict = {"type": "string"} + for _ in range(200): + nested = {"type": "object", "properties": {"x": nested}} + schema = {"type": "object", "properties": {"root": nested}} + + with pytest.raises(ScaleDownError, match="nests deeper"): + _extract_body(config, schema) + + +def test_more_than_one_image_is_rejected(config): + image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGVsbG8="}} + + with pytest.raises(ScaleDownError, match="text only"): + _build_decisions(config, [{"role": "user", "content": [image, image]}], {"questions": {"q": {"type": "noul"}}}) + + +def test_stream_options_is_accepted_and_not_forwarded(config): + assert ( + config.map_openai_params( + non_default_params={"stream_options": {"include_usage": True}}, + optional_params={}, + model="scaledown/summarize", + drop_params=False, + ) + == {} + ) + + +@pytest.mark.parametrize("wrapper", [None, "anyOf", "oneOf"]) +def test_extract_schema_follows_local_refs(config: ScaleDownChatConfig, wrapper: str | None) -> None: + address: Final = {"$ref": "#/$defs/Address"} + schema: Final = { + "type": "object", + "$defs": { + "Address": {"type": "object", "properties": {"city": {"type": "string", "description": "city name"}}} + }, + "properties": { + "address": {wrapper: [address, {"type": "null"}]} if wrapper else address, + "name": {"type": "string"}, + }, + } + + body = config.transform_request( + model="scaledown/extract", + messages=[{"role": "user", "content": "text"}], + optional_params={"response_format": {"type": "json_schema", "json_schema": {"name": "n", "schema": schema}}}, + litellm_params={}, + headers={}, + ) + + assert body["entities"] == {"address": {"city": "city name"}, "name": "name"} + + +@pytest.mark.parametrize( + ("definition", "reference"), + [ + ("Invoice/Details", "#/$defs/Invoice~1Details"), + ("Invoice~Details", "#/$defs/Invoice~0Details"), + ("Invoice~1 Details", "#/%24defs/Invoice~01%20Details"), + ], +) +def test_extract_preserves_fields_from_escaped_refs( + config: ScaleDownChatConfig, definition: str, reference: str +) -> None: + schema: Final = { + "type": "object", + "$defs": {definition: {"type": "object", "properties": {"amount": {"type": "number"}}}}, + "properties": {"invoice": {"$ref": reference}}, + } + + assert _extract_body(config, schema)["entities"] == {"invoice": {"amount": "amount"}} + + +@pytest.mark.parametrize("reference", ["#/$defs/Missing", "https://example.com/schema", "#named-anchor", None, []]) +def test_extract_rejects_unresolvable_refs(config: ScaleDownChatConfig, reference: JsonValue) -> None: + schema: Final = {"type": "object", "properties": {"invoice": {"$ref": reference}}} + + with pytest.raises(ScaleDownError, match=r"\$ref") as exc: + _extract_body(config, schema) + + assert exc.value.status_code == 400 + + +@pytest.mark.parametrize("field_type", ["array", ["array", "null"]]) +def test_extract_nullable_array_keeps_item_fields(config: ScaleDownChatConfig, field_type: str | list[str]) -> None: + schema: Final = { + "type": "object", + "properties": { + "invoices": { + "type": field_type, + "items": {"type": "object", "properties": {"amount": {"type": "number"}}}, + } + }, + } + + assert _extract_body(config, schema)["entities"] == {"invoices": [{"amount": "amount"}]} + + +@pytest.mark.parametrize("wrapper", ["anyOf", "oneOf", "$ref"]) +def test_extract_rejects_sibling_fields_instead_of_dropping_them(config: ScaleDownChatConfig, wrapper: str) -> None: + address: Final = {"type": "object", "properties": {"city": {"type": "string"}}} + reference: Final = {"$ref": "#/$defs/Address"} + field: Final = { + **(reference if wrapper == "$ref" else {wrapper: [address, {"type": "null"}]}), + "properties": {"postal_code": {"type": "string"}}, + } + schema: Final = {"$defs": {"Address": address}, "type": "object", "properties": {"address": field}} + + with pytest.raises(ScaleDownError, match="sibling") as exc: + _extract_body(config, schema) + + assert exc.value.status_code == 400 + + +@pytest.mark.parametrize( + "field", + [ + {"anyOf": [{"type": "object", "properties": {"x": {"type": "string"}}}, {"type": "string"}]}, + {"oneOf": [{"type": "string"}]}, + {"anyOf": [False, {"type": "string"}]}, + {"allOf": [{"type": "object", "properties": {"x": {"type": "string"}}}]}, + ], +) +def test_extract_rejects_unsupported_composition(config: ScaleDownChatConfig, field: dict[str, JsonValue]) -> None: + with pytest.raises(ScaleDownError) as exc: + _extract_body(config, {"type": "object", "properties": {"field": field}}) + + assert exc.value.status_code == 400 + + +@respx.mock +@pytest.mark.parametrize("question_type", [[], {}]) +def test_malformed_question_types_are_request_errors(question_type: list[JsonValue] | dict[str, JsonValue]) -> None: + with pytest.raises(litellm.BadRequestError, match="Question 'q'"): + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + questions={"q": {"type": question_type}}, + ) + + assert not respx.calls + + +@respx.mock +def test_unknown_scaledown_model_is_a_request_error() -> None: + with pytest.raises(litellm.BadRequestError, match="Unknown ScaleDown model"): + litellm.completion(model="scaledown/unknown", messages=[{"role": "user", "content": "text"}]) + + assert not respx.calls + + +def _extract_body(config: ScaleDownChatConfig, schema: dict) -> dict: + return config.transform_request( + model="scaledown/extract", + messages=[{"role": "user", "content": "text"}], + optional_params={"response_format": {"type": "json_schema", "json_schema": {"name": "n", "schema": schema}}}, + litellm_params={}, + headers={}, + ) + + +def test_recursive_schema_is_cut_at_the_cycle_instead_of_expanded(config): + children = {name: {"$ref": "#/$defs/Node"} for name in ("a", "b", "c", "d")} + schema = { + "type": "object", + "$defs": {"Node": {"type": "object", "properties": children}}, + "properties": {"root": {"$ref": "#/$defs/Node"}}, + } + + entities = _extract_body(config, schema)["entities"] + + assert entities == {"root": {"a": "a", "b": "b", "c": "c", "d": "d"}} + + +def test_heavily_repeated_acyclic_refs_are_rejected_by_the_entity_budget(config): + defs = { + f"L{level}": {"type": "object", "properties": {name: {"$ref": f"#/$defs/L{level + 1}"} for name in "abcd"}} + for level in range(10) + } + defs["L10"] = {"type": "string", "description": "leaf"} + schema = {"type": "object", "$defs": defs, "properties": {"root": {"$ref": "#/$defs/L0"}}} + + with pytest.raises(ScaleDownError, match="expands to more than"): + _extract_body(config, schema) + + +def test_summarize_response_is_the_native_payload(config): + payload = {"summary": "terse summary", "input_chars": 400, "output_chars": 13, "input_tokens": 78} + + response = _transform_response(config, "scaledown/summarize", payload) + + assert json.loads(response.choices[0].message.content) == payload + assert response.usage.prompt_tokens == 78 + + +def test_compress_response_reports_top_level_input_tokens(config): + payload = { + "results": {"success": True, "compressed_prompt": "short", "original_prompt_tokens": 52}, + "input_tokens": 52, + } + + response = _transform_response(config, "scaledown/compress", payload) + + assert json.loads(response.choices[0].message.content) == payload + assert response.usage.prompt_tokens == 52 + + +def test_compress_response_falls_back_to_original_prompt_tokens(config): + payload = {"results": {"compressed_prompt": "short", "original_prompt_tokens": 900}} + + assert _transform_response(config, "scaledown/compress", payload).usage.prompt_tokens == 900 + + +def test_extract_request_maps_schema_to_entities_including_nesting(config): + schema = { + "type": "object", + "properties": { + "vendor": {"type": "string", "description": "company that issued the invoice"}, + "notes": {"type": "string"}, + "address": {"type": "object", "properties": {"city": {"type": "string", "description": "city name"}}}, + "line_items": { + "type": "array", + "items": {"type": "object", "properties": {"sku": {"type": "string", "description": "item code"}}}, + }, + }, + } + + body = config.transform_request( + model="scaledown/extract", + messages=[{"role": "user", "content": "Invoice from Northwind."}], + optional_params={"response_format": {"type": "json_schema", "json_schema": {"name": "i", "schema": schema}}}, + litellm_params={}, + headers={}, + ) + + assert body == { + "text": "Invoice from Northwind.", + "entities": { + "vendor": "company that issued the invoice", + "notes": "notes", + "address": {"city": "city name"}, + "line_items": [{"sku": "item code"}], + }, + } + + +def test_extract_without_a_schema_is_rejected(config): + with pytest.raises(ScaleDownError, match="response_format"): + config.transform_request( + model="scaledown/extract", + messages=[{"role": "user", "content": "text"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + +@pytest.mark.parametrize("role", ["system", "developer"]) +def test_summarize_request_splits_instructions_text_and_max_tokens( + config: ScaleDownChatConfig, role: Literal["system", "developer"] +) -> None: + body: Final = config.transform_request( + model="scaledown/summarize", + messages=[ + {"role": role, "content": "Be terse."}, + {"role": "user", "content": "a long document"}, + ], + optional_params={"max_tokens": 40}, + litellm_params={}, + headers={}, + ) + + assert body == {"text": "a long document", "instructions": "Be terse.", "max_tokens": 40} + + +def test_compress_request_puts_earlier_messages_in_context_and_defaults_rate_to_auto(config): + body = config.transform_request( + model="scaledown/compress", + messages=[ + {"role": "system", "content": "background"}, + {"role": "user", "content": "the question"}, + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert body == {"context": "background", "prompt": "the question", "scaledown": {"rate": "auto"}} + + +def test_decisions_response_ignores_the_upstream_cost_field(config): + payload = { + "model": "classify-1", + "answers": { + "category": { + "type": "choice", + "choice": "billing", + "probabilities": {"billing": 0.93, "technical": 0.07}, + "confidence": 0.93, + } + }, + "usage": {"input_tokens": 62, "output_tokens": 1, "cost": 0.00543}, + } + + response = _transform_response(config, "scaledown/decisions", payload) + + assert json.loads(response.choices[0].message.content) == payload["answers"] + assert response.usage.prompt_tokens == 62 + assert response.usage.completion_tokens == 1 + assert "response_cost" not in response._hidden_params + assert response._hidden_params["scaledown_response"] == payload + + +def test_score_answer_survives_the_envelope_intact(config): + answer = { + "type": "score", + "score": 1.43, + "confidence": 0.57, + "legend": {"0": "Cosmetic", "1": "Workaround", "2": "Blocking"}, + "probabilities": {"0": 0.0, "1": 0.57, "2": 0.43}, + } + payload = { + "model": "classify-1", + "answers": {"bug_severity": answer}, + "usage": {"input_tokens": 68, "output_tokens": 1, "cost": 0.0000029}, + } + + response = _transform_response(config, "scaledown/decisions", payload) + + assert json.loads(response.choices[0].message.content)["bug_severity"] == answer + + +def test_decisions_response_without_answers_is_an_error(config): + with pytest.raises(ScaleDownError, match="no answers"): + _transform_response(config, "scaledown/decisions", {"model": "classify-1"}) + + +def test_empty_native_response_is_an_error(config): + with pytest.raises(ScaleDownError, match="empty or malformed"): + _transform_response(config, "scaledown/extract", {}) + + +@pytest.mark.parametrize("upstream_status", [200, 502]) +def test_non_json_response_is_an_upstream_error(config: ScaleDownChatConfig, upstream_status: int) -> None: + raw_response: Final = httpx.Response( + status_code=upstream_status, + text="bad gateway", + request=httpx.Request("POST", f"{BASE}/v1/scaledown"), + ) + + with pytest.raises(ScaleDownError) as exc: + config.transform_response( + model="scaledown/decisions", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=None, + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert exc.value.status_code == 502 + + +def test_decisions_takes_no_openai_sampling_params(config): + assert config.get_supported_openai_params("scaledown/decisions") == ["stream"] + + +def test_unsupported_param_is_rejected_unless_dropped(config): + with pytest.raises(ScaleDownError, match="does not support"): + config.map_openai_params( + non_default_params={"temperature": 0.7}, + optional_params={}, + model="scaledown/decisions", + drop_params=False, + ) + + assert ( + config.map_openai_params( + non_default_params={"temperature": 0.7}, + optional_params={}, + model="scaledown/decisions", + drop_params=True, + ) + == {} + ) + + +@respx.mock +def test_completion_sends_x_api_key_to_the_native_summarize_endpoint(): + route = respx.post(f"{BASE}/summarization/abstractive").mock( + return_value=httpx.Response(200, json={"summary": "s", "input_tokens": 120}) + ) + + response = litellm.completion( + model="scaledown/summarize", + messages=[ + {"role": "system", "content": "Be terse."}, + {"role": "user", "content": "a long document"}, + ], + ) + + request = route.calls[0].request + assert request.headers["x-api-key"] == "sk-scaledown-test" + assert "authorization" not in request.headers + assert json.loads(request.content) == {"text": "a long document", "instructions": "Be terse."} + assert json.loads(response.choices[0].message.content)["summary"] == "s" + + +@respx.mock +def test_completion_routes_decisions_to_the_decisions_endpoint(): + route = respx.post(f"{BASE}/v1/scaledown").mock( + return_value=httpx.Response( + 200, + json={ + "model": "classify-1", + "answers": { + "category": { + "type": "choice", + "choice": "billing", + "probabilities": {"billing": 0.93, "technical": 0.07}, + "confidence": 0.93, + } + }, + "usage": {"input_tokens": 62, "output_tokens": 1, "cost": 0.00543}, + }, + ) + ) + + response = litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "I was charged twice."}], + questions={ + "category": { + "type": "choice", + "criteria": {"billing": "A charge or refund.", "technical": "A bug."}, + } + }, + ) + + sent = json.loads(route.calls[0].request.content) + assert sent["model"] == "classify-1" + assert sent["state"] == {"text": "I was charged twice."} + assert "messages" not in sent + assert json.loads(response.choices[0].message.content)["category"]["choice"] == "billing" + + +@respx.mock +def test_completion_extract_converts_response_format_to_entities(): + schema = { + "type": "json_schema", + "json_schema": { + "name": "invoice", + "schema": { + "type": "object", + "properties": {"vendor": {"type": "string", "description": "company name"}}, + }, + }, + } + extract_result = {"entities": [], "structured_result": {"vendor": "Acme Corp"}, "input_tokens": 20} + route = respx.post(f"{BASE}/extract").mock(return_value=httpx.Response(200, json=extract_result)) + + response = litellm.completion( + model="scaledown/extract", + messages=[{"role": "user", "content": "Acme Corp invoiced $500."}], + response_format=schema, + ) + + assert json.loads(route.calls[0].request.content) == { + "text": "Acme Corp invoiced $500.", + "entities": {"vendor": "company name"}, + } + assert json.loads(response.choices[0].message.content) == {"vendor": "Acme Corp"} + assert response._hidden_params["scaledown_response"] == extract_result + + +@respx.mock +def test_completion_cost_comes_from_input_tokens_not_upstream_usage_cost(): + respx.post(f"{BASE}/v1/scaledown").mock( + return_value=httpx.Response( + 200, + json={ + "model": "classify-1", + "answers": {"q": {"type": "noul", "noul": 0.8}}, + "usage": {"input_tokens": 1000, "output_tokens": 1, "cost": 99.0}, + }, + ) + ) + + response = litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + questions={"q": {"type": "noul", "instructions": "Positive?"}}, + ) + + cost = litellm.completion_cost(completion_response=response, model="scaledown/decisions") + assert cost == pytest.approx(1000 * litellm.model_cost["scaledown/decisions"]["input_cost_per_token"]) + + +@respx.mock +def test_completion_honors_scaledown_api_base(monkeypatch): + monkeypatch.setenv("SCALEDOWN_API_BASE", "https://staging.scaledown.xyz/v1") + route = respx.post("https://staging.scaledown.xyz/v1/scaledown").mock( + return_value=httpx.Response( + 200, + json={ + "model": "classify-1", + "answers": {"q": {"type": "noul", "noul": 0.8}}, + "usage": {"input_tokens": 10, "output_tokens": 1, "cost": 0.0}, + }, + ) + ) + + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + questions={"q": {"type": "noul", "instructions": "Positive?"}}, + ) + + assert route.called + + +@respx.mock +def test_upstream_payment_error_surfaces_the_detail(): + respx.post(f"{BASE}/v1/scaledown").mock(return_value=httpx.Response(402, json={"detail": "Insufficient credits"})) + + with pytest.raises(litellm.exceptions.BadRequestError) as exc: + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + questions={"q": {"type": "noul", "instructions": "Positive?"}}, + ) + + assert "Insufficient credits" in str(exc.value) + + +@respx.mock +def test_failed_completion_invokes_post_api_logger() -> None: + events: Final = SimpleQueue[str]() + + def logger_fn(details: Mapping[str, object]) -> None: + if details.get("log_event_type") == "post_api_call": + events.put(str(details.get("original_response"))) + + respx.post(f"{BASE}/v1/scaledown").respond(402, json={"detail": "Insufficient credits"}) + + with pytest.raises(litellm.BadRequestError): + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + questions={"q": {"type": "noul", "instructions": "Positive?"}}, + logger_fn=logger_fn, + ) + + assert not events.empty() + assert "Insufficient credits" in events.get_nowait() + + +@respx.mock +def test_malformed_decisions_request_never_reaches_the_network(): + route = respx.post(f"{BASE}/v1/scaledown") + + with pytest.raises(Exception, match="questions"): + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + ) + + assert not route.called + + +@respx.mock +def test_questions_passed_through_extra_body_reach_scaledown(): + route = respx.post(f"{BASE}/v1/scaledown").mock( + return_value=httpx.Response( + 200, + json={ + "model": "classify-1", + "answers": {"q": {"type": "noul", "noul": 0.8}}, + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + ) + ) + questions = {"q": {"type": "noul", "instructions": "Positive?"}} + + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + extra_body={"questions": questions}, + ) + + sent = json.loads(route.calls[0].request.content) + assert sent == {"model": DECISIONS_UPSTREAM_MODEL, "state": {"text": "text"}, "questions": questions} + + +@respx.mock +def test_streaming_request_is_served_as_a_single_chunk(): + respx.post(f"{BASE}/compress/raw/").mock( + return_value=httpx.Response(200, json={"compressed_prompt": "c", "original_prompt_tokens": 9}) + ) + + chunks = list( + litellm.completion( + model="scaledown/compress", + messages=[{"role": "user", "content": "the question"}], + stream=True, + ) + ) + + assert "compressed_prompt" in "".join(chunk.choices[0].delta.content or "" for chunk in chunks) + + +@pytest.mark.parametrize( + "structured,expected", + [ + (None, {"vendor": "Northwind"}), + ( + {"invoice": {"amount": 500, "amount_span_anchor": "$500"}}, + {"vendor": "Northwind", "invoice": {"amount": 500}}, + ), + ], +) +def test_extract_merges_first_scalar_match_with_nested_fields(config, structured, expected): + payload = { + "entities": [ + {"type": "vendor", "text": "Northwind", "confidence": 1.0}, + {"type": "vendor", "text": "Contoso", "confidence": 0.8}, + {"type": "unrequested", "text": "ignore"}, + ], + "structured_result": structured, + "input_tokens": 161, + } + response = _transform_response( + config, + "scaledown/extract", + payload, + {"entities": {"vendor": "company name", "missing": "absent field", "invoice": {"amount": "total"}}}, + ) + + assert json.loads(response.choices[0].message.content) == expected + assert response._hidden_params["scaledown_response"] == payload + assert response.usage.prompt_tokens == 161 + + +@respx.mock +def test_strict_extraction_is_rejected_before_network(): + route = respx.post(f"{BASE}/extract") + with pytest.raises(litellm.BadRequestError, match="strict JSON schemas"): + litellm.completion( + model="scaledown/extract", + messages=[{"role": "user", "content": "Invoice from Northwind. Total: $500."}], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "invoice", + "strict": True, + "schema": {"type": "object", "properties": {"amount": {"type": "string"}}, "required": ["amount"]}, + }, + }, + ) + assert not route.called + + +@pytest.mark.parametrize("model", ["classify", "decisions", "extract", "summarize", "compress"]) +@pytest.mark.parametrize( + "content", + [ + [{"type": "image_url", "image_url": {"url": "data:image/png;base64,aGVsbG8="}}], + [ + {"type": "text", "text": "invoice"}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,aGVsbG8="}}, + ], + ], +) +def test_multimodal_inputs_are_rejected_instead_of_dropped(config, model, content): + with pytest.raises(ScaleDownError, match="text only"): + config.transform_request(f"scaledown/{model}", [{"role": "user", "content": content}], {}, {}, {}) + + +@pytest.mark.parametrize("params", [{"document": "QQ=="}, {"state": {"document": "QQ=="}}, {"text": "override"}]) +def test_native_input_overrides_are_rejected(config, params): + with pytest.raises(ScaleDownError, match="text through messages only"): + config.transform_request("scaledown/decisions", [{"role": "user", "content": "text"}], params, {}, {}) + + +@pytest.mark.parametrize("model", ["classify", "decisions", "extract"]) +def test_unsupported_system_instructions_are_rejected(config, model): + with pytest.raises(ScaleDownError, match="does not support system messages"): + config.transform_request( + f"scaledown/{model}", + [{"role": "system", "content": "instructions"}, {"role": "user", "content": "text"}], + {}, + {}, + {}, + ) + + +@respx.mock +@pytest.mark.parametrize("nested", [False, True]) +def test_classify_uses_native_labels_and_preserves_scores(nested): + payload = {"top_label": "urgent", "scores": {"urgent": 0.9, "routine": 0.1}, "input_tokens": 521} + route = respx.post(f"{BASE}/classify").mock(return_value=httpx.Response(200, json=payload)) + labels = [ + {"name": "urgent", "rubric": "Service is unavailable."}, + {"name": "routine", "rubric": "A cosmetic issue."}, + ] + options = {"extra_body": {"labels": labels}} if nested else {"labels": labels} + + response = litellm.completion( + model="scaledown/classify", + messages=[{"role": "user", "content": [{"type": "text", "text": "Server is down."}]}], + **options, + ) + + assert json.loads(route.calls[0].request.content) == {"text": "Server is down.", "labels": labels} + assert json.loads(response.choices[0].message.content) == payload + assert response.usage.prompt_tokens == 521 + assert litellm.completion_cost(completion_response=response) == pytest.approx( + 521 * litellm.model_cost["scaledown/classify"]["input_cost_per_token"] + ) + + +@respx.mock +@pytest.mark.parametrize("labels", [None, [], ["urgent"], [{"name": "urgent"}]]) +def test_classify_requires_labels_before_network(labels): + route = respx.post(f"{BASE}/classify") + with pytest.raises(litellm.BadRequestError, match=r"labels|rubric"): + litellm.completion( + model="scaledown/classify", messages=[{"role": "user", "content": "text"}], extra_body={"labels": labels} + ) + assert not route.called + + +@pytest.mark.asyncio +@respx.mock +async def test_async_extraction_keeps_flat_fields_and_usage(): + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + route = respx.post(f"{BASE}/extract").mock( + return_value=httpx.Response( + 200, + json={ + "entities": [{"type": "vendor", "text": "Northwind"}], + "structured_result": None, + "input_tokens": 161, + }, + ) + ) + client = AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport()) + try: + response = await litellm.acompletion( + model="scaledown/extract", + messages=[{"role": "user", "content": "Invoice from Northwind."}], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "invoice", + "strict": False, + "schema": {"properties": {"vendor": {"type": "string"}}}, + }, + }, + client=client, + ) + finally: + await client.close() + assert json.loads(response.choices[0].message.content) == {"vendor": "Northwind"} + assert response.usage.prompt_tokens == 161 + assert json.loads(route.calls[0].request.content) == { + "text": "Invoice from Northwind.", + "entities": {"vendor": "vendor"}, + } + + +@respx.mock +def test_router_stream_includes_usage_without_sending_stream_options_upstream(): + route = respx.post(f"{BASE}/summarization/abstractive").mock( + return_value=httpx.Response(200, json={"summary": "Fixed billing.", "input_tokens": 90}) + ) + router = litellm.Router( + model_list=[ + {"model_name": "summary", "litellm_params": {"model": "scaledown/summarize", "api_key": "test-key"}} + ], + num_retries=0, + ) + chunks = list( + router.completion( + model="summary", + messages=[{"role": "user", "content": "Billing was fixed on Monday."}], + stream=True, + stream_options={"include_usage": True}, + ) + ) + assert ( + json.loads("".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices))["summary"] + == "Fixed billing." + ) + assert chunks[-1].usage.prompt_tokens == 90 + assert json.loads(route.calls[0].request.content) == {"text": "Billing was fixed on Monday."} + + +@respx.mock +def test_pydantic_response_format_does_not_silently_claim_strict_support(): + from pydantic import BaseModel, ConfigDict + + class Invoice(BaseModel): + model_config = ConfigDict(frozen=True) + vendor: str + + route = respx.post(f"{BASE}/extract") + with pytest.raises(litellm.BadRequestError, match="strict JSON schemas"): + litellm.completion( + model="scaledown/extract", + messages=[{"role": "user", "content": "Invoice from Northwind."}], + response_format=Invoice, + ) + assert not route.called diff --git a/tests/unit/test_scaledown_model_prices.py b/tests/unit/test_scaledown_model_prices.py new file mode 100644 index 00000000000..02c7983f753 --- /dev/null +++ b/tests/unit/test_scaledown_model_prices.py @@ -0,0 +1,46 @@ +from typing import Final + +import pytest + +import litellm +from litellm.types.utils import Usage + +SCALEDOWN_MODELS: Final = ( + "scaledown/classify", + "scaledown/decisions", + "scaledown/extract", + "scaledown/summarize", + "scaledown/compress", +) + +# Source: ScaleDown's standard rate of $0.05 per million input tokens with output unbilled, +# as shown in the ScaleDown usage dashboard (checked 2026-10-09) and its billing service. +INPUT_COST_PER_MILLION_TOKENS: Final = 0.05 + + +@pytest.mark.parametrize("model", SCALEDOWN_MODELS) +def test_scaledown_model_is_priced_on_input_tokens_only(model: str) -> None: + entry: Final = litellm.model_cost[model] + + assert entry["litellm_provider"] == "scaledown" + assert entry["mode"] == "chat" + assert entry["input_cost_per_token"] * 1_000_000 == pytest.approx(INPUT_COST_PER_MILLION_TOKENS) + assert entry["output_cost_per_token"] == 0.0 + + +@pytest.mark.parametrize("model", SCALEDOWN_MODELS) +def test_scaledown_cost_ignores_output_tokens(model: str) -> None: + prompt_cost, completion_cost = litellm.cost_per_token( + model=model, + custom_llm_provider="scaledown", + usage_object=Usage(prompt_tokens=1_000_000, completion_tokens=500_000), + ) + + assert prompt_cost == pytest.approx(INPUT_COST_PER_MILLION_TOKENS) + assert completion_cost == 0.0 + + +def test_every_scaledown_entry_in_the_cost_map_is_covered_here() -> None: + in_map: Final = {key for key in litellm.model_cost if key.startswith("scaledown/")} + + assert in_map == set(SCALEDOWN_MODELS) From b6037990c258380ea95572ded12927eb34f8deba Mon Sep 17 00:00:00 2001 From: Soham Chatterjee <96soham96@gmail.com> Date: Sat, 10 Oct 2026 23:02:03 +0200 Subject: [PATCH 17/40] feat(models): register ScaleDown text pricing (#44167) * Add ScaleDown model pricing Registers the five ScaleDown models in the pricing map. ScaleDown bills on input tokens only, so output_cost_per_token is zero for every entry. The classify and decisions models bill at $0.04 per million input tokens, and extract, summarize, and compress bill at $0.05 per million. Co-Authored-By: Claude Opus 5 * Price all ScaleDown models at $0.04 per million and drop unverified context limits Every ScaleDown model now bills $0.04 per million input tokens. The 32000-token max_input_tokens/max_output_tokens values were not confirmed against the service, so they are removed rather than registered. Co-Authored-By: Claude Sonnet 5.5 * Price all ScaleDown models at $0.05 per million input tokens This is the standard rate in ScaleDown's billing and matches the usage dashboard. The Decisions usage.cost field is not used for pricing. Co-Authored-By: Claude Sonnet 5.5 * Add regression tests for the ScaleDown pricing entries Co-Authored-By: Claude Sonnet 5.5 * Cite the source of the ScaleDown rate in the pricing tests and type them Co-Authored-By: Claude Sonnet 5.5 * fix(scaledown): type the pricing model registry * fix(scaledown): address pricing review annotations --------- Co-authored-by: Claude Opus 5 Co-authored-by: moe-berri From 9e7e5c24e4315ca8b5d87d2c30de64b5fb816aa1 Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 14:02:16 -0700 Subject: [PATCH 18/40] refactor(types): move UserAPIKeyAuth, its token view and hash_token into litellm.types.proxy.auth.user_api_key_auth (#45800) * refactor(types): move LitellmUserRoles and KeyManagementRoutes into litellm.types.proxy.auth.user_roles Part of LIT-9423 * refactor(types): move is_jwt into litellm.types.proxy.auth.user_api_key_auth Part of LIT-9423 * refactor(auth): alias is_jwt import so JWTHandler.is_jwt is not read as recursive * refactor(types): type UserAPIKeyAuth, its token view and hash_token in place Part of LIT-9423 * refactor(types): move UserAPIKeyAuth, its token view and hash_token into litellm.types.proxy.auth.user_api_key_auth Part of LIT-9423 * fix(types): keep agent, budget snapshot and is_jwt names importable from proxy._types Part of LIT-9423 * fix(types): restore main's _types.py content alongside the user_api_key_auth move Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * style(types): drop blank lines between proxy._types re-export imports Part of LIT-9423 * refactor(proxy): keep only user_api_key_auth re-exports in _types.py Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): carry main's UserAPIKeyAuth field into moved file Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(types): drop blank lines between proxy._types re-export imports Part of LIT-9423 --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 335 ++---------------- litellm/types/proxy/auth/user_api_key_auth.py | 331 ++++++++++++++++- .../test_key_management_endpoints.py | 6 +- 3 files changed, 361 insertions(+), 311 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 4abb93454c7..7b2a63974d8 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -16,7 +16,6 @@ from pydantic import ( Json, JsonValue, PositiveInt, - PrivateAttr, TypeAdapter, field_validator, model_validator, @@ -39,7 +38,12 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( validate_no_callback_env_reference, ) from litellm.proxy._experimental.mcp_server.stdio_gate import MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_enabled -from litellm.types.agents import AgentCaller, AgentResponse +from litellm.types.agents import ( + AgentCaller as AgentCaller, # noqa: PLC0414 # public re-export +) +from litellm.types.agents import ( + AgentResponse as AgentResponse, # noqa: PLC0414 # public re-export +) from litellm.types.integrations.compression_interception import ( CompressionSavingsMetadata, ) @@ -68,11 +72,17 @@ from litellm.types.proxy.agent_identity import ManagedAgentContext from litellm.types.proxy.auth.special_headers import ( SpecialHeaders as SpecialHeaders, # noqa: PLC0414 # public re-export ) -from litellm.types.proxy.auth.user_api_key_auth import is_jwt +from litellm.types.proxy.auth.user_api_key_auth import ( + is_jwt as is_jwt, # noqa: PLC0414 # public re-export +) from litellm.types.proxy.carried_budget_state import ( - OrgBudgetSnapshot, - TeamBudgetSnapshot, - UserBudgetSnapshot, + OrgBudgetSnapshot as OrgBudgetSnapshot, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.carried_budget_state import ( + TeamBudgetSnapshot as TeamBudgetSnapshot, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.carried_budget_state import ( + UserBudgetSnapshot as UserBudgetSnapshot, # noqa: PLC0414 # public re-export ) from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry from litellm.types.proxy.spend_capture_rate import SpendCaptureRateCheckSettings @@ -198,17 +208,9 @@ class LitellmTableNames(str, enum.Enum): from litellm.types.integrations.slack_alerting import ( Litellm_EntityType as Litellm_EntityType, # noqa: E402, PLC0414 # public re-export ) - - -def hash_token(token: str) -> str: - import hashlib - - # Hash the string using SHA-256 - hashed_token: Final = hashlib.sha256(token.encode()).hexdigest() - - return hashed_token - - +from litellm.types.proxy.auth.user_api_key_auth import ( # noqa: E402 # re-export after the definitions above + hash_token as hash_token, # noqa: PLC0414 # public re-export +) from litellm.types.proxy.auth.user_roles import ( # noqa: E402 # re-export after the definitions above KeyManagementRoutes as KeyManagementRoutes, # noqa: PLC0414 # public re-export ) @@ -3058,296 +3060,15 @@ from litellm.models.verification_token import ( # noqa: E402 from litellm.models.verification_token import ( # noqa: E402 LiteLLM_VerificationToken as LiteLLM_VerificationToken, ) - -_UntypedDict: TypeAlias = dict[Any, Any] - - -def _is_str_keyed_dict(value: object) -> TypeIs[dict[str, object]]: # guard-ok: pydantic input and JSON keys are str - return isinstance(value, dict) - - -class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): - """ - Combined view of litellm verification token + litellm team table (select values) - """ - - team_spend: float | None = None - team_alias: str | None = None - team_tpm_limit: int | None = None - team_rpm_limit: int | None = None - team_tpd_limit: int | None = None - team_max_budget: float | None = None - team_soft_budget: float | None = None - team_model_max_budget: dict[str, object] | None = None - team_models: list[Any] = [] - team_blocked: bool = False - soft_budget: float | None = None - team_model_aliases: _UntypedDict | None = None - team_member: Member | None = None - team_metadata: _UntypedDict | None = None - team_object_permission_id: str | None = None - - # Team Member Specific Params - team_member_spend: float | None = None - team_member_tpm_limit: int | None = None - team_member_rpm_limit: int | None = None - - # End User Params - end_user_id: str | None = None - end_user_tpm_limit: int | None = None - end_user_rpm_limit: int | None = None - end_user_tpd_limit: int | None = None - end_user_max_budget: float | None = None - end_user_model_max_budget: _UntypedDict | None = None - - # Organization Params - organization_alias: str | None = None - organization_max_budget: float | None = None - organization_tpm_limit: int | None = None - organization_rpm_limit: int | None = None - organization_metadata: _UntypedDict | None = None - - # Project Params - project_alias: str | None = None - project_metadata: _UntypedDict | None = None - - # Time stamps - last_refreshed_at: float | None = None # last time joint view was pulled from db - - def __init__(self, **kwargs: object) -> None: - # Handle litellm_budget_table_* keys (budget table overrides when key value is None or empty) - for key, value in list(kwargs.items()): - if key.startswith("litellm_budget_table_") and value is not None: - # Extract the corresponding attribute name - attr_name = key.replace("litellm_budget_table_", "") - # Use key's value from kwargs (from DB view), not class default - current = kwargs.get(attr_name) - if current is None: - current = getattr(self, attr_name, None) - # Apply budget value when key has no value, or for model_max_budget when key has empty dict - should_apply = current is None or ( - attr_name == "model_max_budget" and _is_str_keyed_dict(current) and len(current) == 0 - ) - if should_apply: - kwargs[attr_name] = value - if key == "end_user_id" and value is not None and isinstance(value, int): - kwargs[key] = str(value) - - if kwargs.get("organization_id") is not None: - kwargs["org_id"] = kwargs.pop("organization_id") - # Initialize the superclass - super().__init__(**kwargs) # pyright: ignore[reportArgumentType, reportUnknownMemberType] # pydantic validates raw values - - -class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response object for user api key auth - """ - Return the row in the db - """ - - api_key: str | None = None - user_role: LitellmUserRoles | None = None - allowed_model_region: AllowedModelRegion | None = None - parent_otel_span: Span | None = None - rpm_limit_per_model: dict[str, int] | None = None - tpm_limit_per_model: dict[str, int] | None = None - user_tpm_limit: int | None = None - user_rpm_limit: int | None = None - user_email: str | None = None - user_spend: float | None = None - user_max_budget: float | None = None - # Values stay `object` rather than BudgetConfig: this is the raw JSON column, - # and validating it here would make one malformed row fail auth outright. - # resolve_model_budget validates the single entry a request actually needs. - user_model_max_budget: Mapping[str, object] | None = None - team_member_model_max_budget: Mapping[str, object] | None = Field(default=None, exclude=True) - request_route: str | None = None - is_session_token: bool = False - # Server-only marker set exclusively by the MCP gateway admission path - # (reload_admitted_user) for a keyless user-subject admitted via a gateway DCR session - # bearer or bridge envelope. Not a DB column and never populated from caller-controlled key - # metadata or JWT claims, so it cannot be forged to gain the team-inherited MCP grant union - # or to escape the caller-Authorization egress scrub. exclude=True keeps it out of serialization. - mcp_admitted_user_subject: bool = Field(default=False, exclude=True) - requires_fresh_policy: bool = Field(default=False, exclude=True) - mcp_explicit_grants_only: bool = Field(default=False, exclude=True) - # team_id -> that team's mcp_rpm_limit map, for a keyless admitted subject that reaches MCP - # servers through several teams at once and therefore has no single team_id for the limiter to - # key off. Server-only and stripped from validated input for the same reason as the marker - # above: a forged entry would let a caller pick which team's rpm bucket it is charged against. - mcp_source_team_rpm_limits: dict[str, dict[str, int]] | None = Field(default=None, exclude=True) - # The single MCP server_id a gateway session bearer was scoped to at authorize time (RFC 8707 - # resource), or None for an aggregate-scope session. A RESTRICTION intersected against the live - # grant resolution, never a grant. Server-only, set exclusively by the MCP gateway admission - # path via post-construction assignment and stripped from validated input like the markers - # above; a forged value could at most narrow, but the stripping keeps the field's provenance - # single-owner so its meaning stays trustworthy. - mcp_session_resource_server_id: str | None = Field(default=None, exclude=True) - mcp_toolset_id: str | None = Field(default=None, exclude=True) - authenticated_by_custom_auth: bool = Field(default=False, exclude=True) - via_virtual_key: bool = Field( - default=False, - exclude=True, - description=( - "Server-only marker set exclusively by the DB virtual-key and master-key auth paths via " - "post-construction assignment. Stripped from validated input so custom auth handlers, JWT " - "claims, or key metadata cannot forge it. Gates overwrite_user_with_key_hash stamping: only " - "a credential the proxy itself validated as a key may be forwarded as the provider-facing " - "user id." - ), - ) - invoked_agent_id: str | None = Field(default=None, exclude=True) - invoked_agent_policy: AgentResponse | None = Field(default=None, exclude=True) - agent_invocation_cost: float | None = Field(default=None, exclude=True) - billing_agent_policy: AgentResponse | None = Field(default=None, exclude=True) - _managed_delegation_verified: bool = PrivateAttr(default=False) - managed_agent_policy: AgentResponse | None = Field(default=None, exclude=True) - managed_agent_context: ManagedAgentContext | None = Field(default=None, exclude=True) - agent_caller: AgentCaller | None = Field( - default=None, - exclude=True, - description=( - "Set per request from the x-litellm-user-id / x-litellm-team-id headers an agent echoes back on " - "calls made with its own key. Every check treats it as a ceiling, so a forged value can only " - "narrow the agent's access." - ), - ) - budget_reservation: dict[str, Any] | None = Field(default=None, exclude=True) - team_budget_snapshot: TeamBudgetSnapshot | None = Field(default=None, exclude=True) - user_budget_snapshot: UserBudgetSnapshot | None = Field(default=None, exclude=True) - org_budget_snapshot: OrgBudgetSnapshot | None = Field(default=None, exclude=True) - matched_model_access_groups: list[str] | None = Field(default=None, exclude=True) - budget_throttle_pct: float | None = Field(default=None, exclude=True) - user: Any | None = None # Expanded user object when expand=user is used - created_by_user: Any | None = None # Expanded created_by user when expand=user is used - end_user_object_permission: LiteLLM_ObjectPermissionTable | None = None - # Team object_permission preloaded in auth (e.g. get_team_object) to avoid - # per-request object_permission fetches in downstream checks (vector stores, etc.) - team_object_permission: LiteLLM_ObjectPermissionTable | None = None - # Decoded upstream IdP claims (groups, roles, etc.) propagated by JWT auth machinery - # and forwarded into outbound tokens by guardrails such as MCPJWTSigner. - jwt_claims: _UntypedDict | None = None - - model_config = ConfigDict(arbitrary_types_allowed=True) - - @model_validator(mode="before") - @classmethod - def check_api_key(cls, values: object) -> object: - # If values is already an instance (not a dict), return it as-is - if not _is_str_keyed_dict(values): - return values - # mcp_admitted_user_subject is a server-only marker, set ONLY by the MCP gateway admission - # path via post-construction assignment. Strip it from any validated input (constructor - # kwargs, model_validate, a JWT/key claim splat) so it can never be forged from caller data. - values.pop("mcp_admitted_user_subject", None) - values.pop("requires_fresh_policy", None) - values.pop("mcp_explicit_grants_only", None) - values.pop("mcp_source_team_rpm_limits", None) - values.pop("mcp_session_resource_server_id", None) - values.pop("mcp_toolset_id", None) - values.pop("via_virtual_key", None) - values.pop("authenticated_by_custom_auth", None) - values.pop("agent_caller", None) - values.pop("managed_agent_context", None) - values.pop("managed_agent_policy", None) - values.pop("invoked_agent_id", None) - values.pop("invoked_agent_policy", None) - values.pop("agent_invocation_cost", None) - values.pop("billing_agent_policy", None) - api_key: Final = values.get("api_key") - if api_key is not None: - values.update({"token": cls._safe_hash_litellm_api_key(api_key)}) # pyright: ignore[reportArgumentType] # non-str fails in the hasher - if isinstance(api_key, str): - values.update({"api_key": cls._safe_hash_litellm_api_key(api_key)}) - return values - - @classmethod - def _safe_hash_litellm_api_key(cls, api_key: str) -> str: - """ - Helper to ensure all logged keys are hashed - Covers: - 1. Regular API keys from LiteLLM DB - 2. JWT tokens used for connecting to LiteLLM API - """ - normalized = api_key - if normalized[:7].lower() == "bearer ": - normalized = normalized[7:] - if normalized.startswith("sk-"): - return hash_token(normalized) - if is_jwt(token=normalized): - return f"hashed-jwt-{hash_token(token=normalized)}" - return normalized - - @classmethod - def get_litellm_internal_health_check_user_api_key_auth(cls) -> "UserAPIKeyAuth": - """ - Returns a `UserAPIKeyAuth` object for the litellm internal health check service account. - - This is used to track number of requests/spend for health check calls. - """ - from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME - - return cls( - api_key=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, - team_id=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, - key_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, - team_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, - ) - - @classmethod - def get_litellm_cli_user_api_key_auth(cls) -> "UserAPIKeyAuth": - """ - Returns a `UserAPIKeyAuth` object for the litellm internal health check service account. - - This is used to track number of requests/spend for health check calls. - """ - from litellm.constants import LITTELM_CLI_SERVICE_ACCOUNT_NAME - - return cls( - api_key=LITTELM_CLI_SERVICE_ACCOUNT_NAME, - team_id=LITTELM_CLI_SERVICE_ACCOUNT_NAME, - key_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME, - team_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME, - ) - - @classmethod - def get_litellm_internal_jobs_user_api_key_auth(cls) -> "UserAPIKeyAuth": - """ - Returns a `UserAPIKeyAuth` object for internal LiteLLM jobs like key rotation. - - This is used to track actions performed by automated system jobs. - """ - from litellm.constants import LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME - - return cls( - api_key=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, - team_id="system", - key_alias=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, - team_alias="system", - user_id="system", - user_role=LitellmUserRoles.PROXY_ADMIN, - ) - - @property - def is_team_service_account(self) -> bool: - return ( - self.user_id is None - and self.team_id is not None - and bool(self.metadata) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped base field - and self.metadata.get("service_account_id") is not None # pyright: ignore[reportUnknownMemberType] # untyped base field - ) - - -def user_api_key_has_admin_view(user_api_key_dict: UserAPIKeyAuth) -> bool: - """Return True if the caller's role grants unscoped read access to all - tenant resources (managed files, batches, vector stores, spend rows, etc). - - Lives on _types.py so leaf modules (e.g. litellm.llms.base_llm.managed_resources) - can use it without pulling in litellm.proxy.utils via management_endpoints. - """ - return user_api_key_dict.user_role in ( - LitellmUserRoles.PROXY_ADMIN, - LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - ) +from litellm.types.proxy.auth.user_api_key_auth import ( # noqa: E402 # re-export after the definitions above + LiteLLM_VerificationTokenView as LiteLLM_VerificationTokenView, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.auth.user_api_key_auth import ( + UserAPIKeyAuth as UserAPIKeyAuth, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.auth.user_api_key_auth import ( + user_api_key_has_admin_view as user_api_key_has_admin_view, # noqa: PLC0414 # public re-export +) class UserInfoResponse(LiteLLMPydanticObjectBase): diff --git a/litellm/types/proxy/auth/user_api_key_auth.py b/litellm/types/proxy/auth/user_api_key_auth.py index 3d879fdcc43..18dd4e53701 100644 --- a/litellm/types/proxy/auth/user_api_key_auth.py +++ b/litellm/types/proxy/auth/user_api_key_auth.py @@ -1,4 +1,33 @@ -from typing import Final +from collections.abc import Mapping +from typing import ( + TYPE_CHECKING, + Any, # noqa: TID251 # moved UserAPIKeyAuth fields and the Span alias are typed Any + Final, + TypeAlias, +) + +from pydantic import ConfigDict, Field, PrivateAttr, model_validator +from typing_extensions import TypeIs # noqa: TID251 # narrows the before-validator input without copying it + +from litellm.models.object_permission import LiteLLM_ObjectPermissionTable +from litellm.models.team import Member +from litellm.models.verification_token import LiteLLM_VerificationToken +from litellm.types.agents import AgentCaller, AgentResponse +from litellm.types.proxy.agent_identity import ManagedAgentContext +from litellm.types.proxy.auth.user_roles import LitellmUserRoles +from litellm.types.proxy.carried_budget_state import ( + OrgBudgetSnapshot, + TeamBudgetSnapshot, + UserBudgetSnapshot, +) +from litellm.types.router import AllowedModelRegion + +if TYPE_CHECKING: + from opentelemetry.trace import Span as _Span + + Span: TypeAlias = _Span | Any +else: + Span: TypeAlias = Any def is_jwt(token: str | None) -> bool: @@ -6,3 +35,303 @@ def is_jwt(token: str | None) -> bool: return False parts: Final = token.split(".") return len(parts) == 3 + + +def hash_token(token: str) -> str: + import hashlib + + # Hash the string using SHA-256 + hashed_token: Final = hashlib.sha256(token.encode()).hexdigest() + + return hashed_token + + +_UntypedDict: TypeAlias = dict[Any, Any] + + +def _is_str_keyed_dict(value: object) -> TypeIs[dict[str, object]]: # guard-ok: pydantic input and JSON keys are str + return isinstance(value, dict) + + +class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): + """ + Combined view of litellm verification token + litellm team table (select values) + """ + + team_spend: float | None = None + team_alias: str | None = None + team_tpm_limit: int | None = None + team_rpm_limit: int | None = None + team_tpd_limit: int | None = None + team_max_budget: float | None = None + team_soft_budget: float | None = None + team_model_max_budget: dict[str, object] | None = None + team_models: list[Any] = [] + team_blocked: bool = False + soft_budget: float | None = None + team_model_aliases: _UntypedDict | None = None + team_member: Member | None = None + team_metadata: _UntypedDict | None = None + team_object_permission_id: str | None = None + + # Team Member Specific Params + team_member_spend: float | None = None + team_member_tpm_limit: int | None = None + team_member_rpm_limit: int | None = None + + # End User Params + end_user_id: str | None = None + end_user_tpm_limit: int | None = None + end_user_rpm_limit: int | None = None + end_user_tpd_limit: int | None = None + end_user_max_budget: float | None = None + end_user_model_max_budget: _UntypedDict | None = None + + # Organization Params + organization_alias: str | None = None + organization_max_budget: float | None = None + organization_tpm_limit: int | None = None + organization_rpm_limit: int | None = None + organization_metadata: _UntypedDict | None = None + + # Project Params + project_alias: str | None = None + project_metadata: _UntypedDict | None = None + + # Time stamps + last_refreshed_at: float | None = None # last time joint view was pulled from db + + def __init__(self, **kwargs: object) -> None: + # Handle litellm_budget_table_* keys (budget table overrides when key value is None or empty) + for key, value in list(kwargs.items()): + if key.startswith("litellm_budget_table_") and value is not None: + # Extract the corresponding attribute name + attr_name = key.replace("litellm_budget_table_", "") + # Use key's value from kwargs (from DB view), not class default + current = kwargs.get(attr_name) + if current is None: + current = getattr(self, attr_name, None) + # Apply budget value when key has no value, or for model_max_budget when key has empty dict + should_apply = current is None or ( + attr_name == "model_max_budget" and _is_str_keyed_dict(current) and len(current) == 0 + ) + if should_apply: + kwargs[attr_name] = value + if key == "end_user_id" and value is not None and isinstance(value, int): + kwargs[key] = str(value) + + if kwargs.get("organization_id") is not None: + kwargs["org_id"] = kwargs.pop("organization_id") + # Initialize the superclass + super().__init__(**kwargs) # pyright: ignore[reportArgumentType, reportUnknownMemberType] # pydantic validates raw values + + +class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response object for user api key auth + """ + Return the row in the db + """ + + api_key: str | None = None + user_role: LitellmUserRoles | None = None + allowed_model_region: AllowedModelRegion | None = None + parent_otel_span: Span | None = None + rpm_limit_per_model: dict[str, int] | None = None + tpm_limit_per_model: dict[str, int] | None = None + user_tpm_limit: int | None = None + user_rpm_limit: int | None = None + user_email: str | None = None + user_spend: float | None = None + user_max_budget: float | None = None + # Values stay `object` rather than BudgetConfig: this is the raw JSON column, + # and validating it here would make one malformed row fail auth outright. + # resolve_model_budget validates the single entry a request actually needs. + user_model_max_budget: Mapping[str, object] | None = None + team_member_model_max_budget: Mapping[str, object] | None = Field(default=None, exclude=True) + request_route: str | None = None + is_session_token: bool = False + # Server-only marker set exclusively by the MCP gateway admission path + # (reload_admitted_user) for a keyless user-subject admitted via a gateway DCR session + # bearer or bridge envelope. Not a DB column and never populated from caller-controlled key + # metadata or JWT claims, so it cannot be forged to gain the team-inherited MCP grant union + # or to escape the caller-Authorization egress scrub. exclude=True keeps it out of serialization. + mcp_admitted_user_subject: bool = Field(default=False, exclude=True) + requires_fresh_policy: bool = Field(default=False, exclude=True) + mcp_explicit_grants_only: bool = Field(default=False, exclude=True) + # team_id -> that team's mcp_rpm_limit map, for a keyless admitted subject that reaches MCP + # servers through several teams at once and therefore has no single team_id for the limiter to + # key off. Server-only and stripped from validated input for the same reason as the marker + # above: a forged entry would let a caller pick which team's rpm bucket it is charged against. + mcp_source_team_rpm_limits: dict[str, dict[str, int]] | None = Field(default=None, exclude=True) + # The single MCP server_id a gateway session bearer was scoped to at authorize time (RFC 8707 + # resource), or None for an aggregate-scope session. A RESTRICTION intersected against the live + # grant resolution, never a grant. Server-only, set exclusively by the MCP gateway admission + # path via post-construction assignment and stripped from validated input like the markers + # above; a forged value could at most narrow, but the stripping keeps the field's provenance + # single-owner so its meaning stays trustworthy. + mcp_session_resource_server_id: str | None = Field(default=None, exclude=True) + mcp_toolset_id: str | None = Field(default=None, exclude=True) + authenticated_by_custom_auth: bool = Field(default=False, exclude=True) + via_virtual_key: bool = Field( + default=False, + exclude=True, + description=( + "Server-only marker set exclusively by the DB virtual-key and master-key auth paths via " + "post-construction assignment. Stripped from validated input so custom auth handlers, JWT " + "claims, or key metadata cannot forge it. Gates overwrite_user_with_key_hash stamping: only " + "a credential the proxy itself validated as a key may be forwarded as the provider-facing " + "user id." + ), + ) + invoked_agent_id: str | None = Field(default=None, exclude=True) + invoked_agent_policy: AgentResponse | None = Field(default=None, exclude=True) + agent_invocation_cost: float | None = Field(default=None, exclude=True) + billing_agent_policy: AgentResponse | None = Field(default=None, exclude=True) + _managed_delegation_verified: bool = PrivateAttr(default=False) + managed_agent_policy: AgentResponse | None = Field(default=None, exclude=True) + managed_agent_context: ManagedAgentContext | None = Field(default=None, exclude=True) + agent_caller: AgentCaller | None = Field( + default=None, + exclude=True, + description=( + "Set per request from the x-litellm-user-id / x-litellm-team-id headers an agent echoes back on " + "calls made with its own key. Every check treats it as a ceiling, so a forged value can only " + "narrow the agent's access." + ), + ) + budget_reservation: dict[str, Any] | None = Field(default=None, exclude=True) + team_budget_snapshot: TeamBudgetSnapshot | None = Field(default=None, exclude=True) + user_budget_snapshot: UserBudgetSnapshot | None = Field(default=None, exclude=True) + org_budget_snapshot: OrgBudgetSnapshot | None = Field(default=None, exclude=True) + matched_model_access_groups: list[str] | None = Field(default=None, exclude=True) + budget_throttle_pct: float | None = Field(default=None, exclude=True) + user: Any | None = None # Expanded user object when expand=user is used + created_by_user: Any | None = None # Expanded created_by user when expand=user is used + end_user_object_permission: LiteLLM_ObjectPermissionTable | None = None + # Team object_permission preloaded in auth (e.g. get_team_object) to avoid + # per-request object_permission fetches in downstream checks (vector stores, etc.) + team_object_permission: LiteLLM_ObjectPermissionTable | None = None + # Decoded upstream IdP claims (groups, roles, etc.) propagated by JWT auth machinery + # and forwarded into outbound tokens by guardrails such as MCPJWTSigner. + jwt_claims: _UntypedDict | None = None + + model_config = ConfigDict(arbitrary_types_allowed=True) + + @model_validator(mode="before") + @classmethod + def check_api_key(cls, values: object) -> object: + # If values is already an instance (not a dict), return it as-is + if not _is_str_keyed_dict(values): + return values + # mcp_admitted_user_subject is a server-only marker, set ONLY by the MCP gateway admission + # path via post-construction assignment. Strip it from any validated input (constructor + # kwargs, model_validate, a JWT/key claim splat) so it can never be forged from caller data. + values.pop("mcp_admitted_user_subject", None) + values.pop("requires_fresh_policy", None) + values.pop("mcp_explicit_grants_only", None) + values.pop("mcp_source_team_rpm_limits", None) + values.pop("mcp_session_resource_server_id", None) + values.pop("mcp_toolset_id", None) + values.pop("via_virtual_key", None) + values.pop("authenticated_by_custom_auth", None) + values.pop("agent_caller", None) + values.pop("managed_agent_context", None) + values.pop("managed_agent_policy", None) + values.pop("invoked_agent_id", None) + values.pop("invoked_agent_policy", None) + values.pop("agent_invocation_cost", None) + values.pop("billing_agent_policy", None) + api_key: Final = values.get("api_key") + if api_key is not None: + values.update({"token": cls._safe_hash_litellm_api_key(api_key)}) # pyright: ignore[reportArgumentType] # non-str fails in the hasher + if isinstance(api_key, str): + values.update({"api_key": cls._safe_hash_litellm_api_key(api_key)}) + return values + + @classmethod + def _safe_hash_litellm_api_key(cls, api_key: str) -> str: + """ + Helper to ensure all logged keys are hashed + Covers: + 1. Regular API keys from LiteLLM DB + 2. JWT tokens used for connecting to LiteLLM API + """ + normalized = api_key + if normalized[:7].lower() == "bearer ": + normalized = normalized[7:] + if normalized.startswith("sk-"): + return hash_token(normalized) + if is_jwt(token=normalized): + return f"hashed-jwt-{hash_token(token=normalized)}" + return normalized + + @classmethod + def get_litellm_internal_health_check_user_api_key_auth(cls) -> "UserAPIKeyAuth": + """ + Returns a `UserAPIKeyAuth` object for the litellm internal health check service account. + + This is used to track number of requests/spend for health check calls. + """ + from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME + + return cls( + api_key=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + team_id=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + key_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + team_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + ) + + @classmethod + def get_litellm_cli_user_api_key_auth(cls) -> "UserAPIKeyAuth": + """ + Returns a `UserAPIKeyAuth` object for the litellm internal health check service account. + + This is used to track number of requests/spend for health check calls. + """ + from litellm.constants import LITTELM_CLI_SERVICE_ACCOUNT_NAME + + return cls( + api_key=LITTELM_CLI_SERVICE_ACCOUNT_NAME, + team_id=LITTELM_CLI_SERVICE_ACCOUNT_NAME, + key_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME, + team_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME, + ) + + @classmethod + def get_litellm_internal_jobs_user_api_key_auth(cls) -> "UserAPIKeyAuth": + """ + Returns a `UserAPIKeyAuth` object for internal LiteLLM jobs like key rotation. + + This is used to track actions performed by automated system jobs. + """ + from litellm.constants import LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME + + return cls( + api_key=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, + team_id="system", + key_alias=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, + team_alias="system", + user_id="system", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + @property + def is_team_service_account(self) -> bool: + return ( + self.user_id is None + and self.team_id is not None + and bool(self.metadata) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped base field + and self.metadata.get("service_account_id") is not None # pyright: ignore[reportUnknownMemberType] # untyped base field + ) + + +def user_api_key_has_admin_view(user_api_key_dict: UserAPIKeyAuth) -> bool: + """Return True if the caller's role grants unscoped read access to all + tenant resources (managed files, batches, vector stores, spend rows, etc). + + Lives on _types.py so leaf modules (e.g. litellm.llms.base_llm.managed_resources) + can use it without pulling in litellm.proxy.utils via management_endpoints. + """ + return user_api_key_dict.user_role in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ) diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 9905ee619c2..308558450b1 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -7817,7 +7817,7 @@ async def test_process_single_key_update(): mock_delete_cache.return_value = None # Mock hash_token (imported from litellm.proxy._types) - with patch("litellm.proxy._types.hash_token") as mock_hash: + with patch("litellm.types.proxy.auth.user_api_key_auth.hash_token") as mock_hash: mock_hash.return_value = "hashed-test-key-123" # Mock _hash_token_if_needed @@ -7963,7 +7963,7 @@ async def test_bulk_update_keys_success(monkeypatch): with patch( "litellm.proxy.management_endpoints.key_management_endpoints.delete_cache_key_object" ): - with patch("litellm.proxy._types.hash_token") as mock_hash: + with patch("litellm.types.proxy.auth.user_api_key_auth.hash_token") as mock_hash: mock_hash.side_effect = ["hashed-key-1", "hashed-key-2"] def _hash_for_bulk_success(token: str) -> str: @@ -8091,7 +8091,7 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): with patch( "litellm.proxy.management_endpoints.key_management_endpoints.delete_cache_key_object" ): - with patch("litellm.proxy._types.hash_token") as mock_hash: + with patch("litellm.types.proxy.auth.user_api_key_auth.hash_token") as mock_hash: mock_hash.return_value = "hashed-key-1" def _hash_for_bulk_partial(token: str) -> str: From c582bc6e68a1b78e9d63d6d63f37ee66f8c976c1 Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 14:02:53 -0700 Subject: [PATCH 19/40] refactor(core_utils): move resource_ownership out of the proxy (#45805) * refactor(types): move LitellmUserRoles and KeyManagementRoutes into litellm.types.proxy.auth.user_roles Part of LIT-9423 * refactor(types): move is_jwt into litellm.types.proxy.auth.user_api_key_auth Part of LIT-9423 * refactor(auth): alias is_jwt import so JWTHandler.is_jwt is not read as recursive * refactor(types): type UserAPIKeyAuth, its token view and hash_token in place Part of LIT-9423 * refactor(types): move UserAPIKeyAuth, its token view and hash_token into litellm.types.proxy.auth.user_api_key_auth Part of LIT-9423 * fix(types): keep agent, budget snapshot and is_jwt names importable from proxy._types Part of LIT-9423 * refactor(core_utils): move resource_ownership out of the proxy Part of LIT-9423 * fix(types): restore main's _types.py content alongside the user_api_key_auth move Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * style(types): drop blank lines between proxy._types re-export imports Part of LIT-9423 * refactor(proxy): keep only user_api_key_auth re-exports in _types.py Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): carry main's UserAPIKeyAuth field into moved file Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(types): drop blank lines between proxy._types re-export imports Part of LIT-9423 --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/resource_ownership.py | 93 +++++++++++++++ .../base_llm/managed_resources/isolation.py | 2 +- litellm/llms/litellm_proxy/skills/handler.py | 12 +- .../proxy/common_utils/resource_ownership.py | 106 +++--------------- .../common_utils/test_resource_ownership.py | 29 +++++ 5 files changed, 146 insertions(+), 96 deletions(-) create mode 100644 litellm/litellm_core_utils/resource_ownership.py create mode 100644 tests/unit/proxy/common_utils/test_resource_ownership.py diff --git a/litellm/litellm_core_utils/resource_ownership.py b/litellm/litellm_core_utils/resource_ownership.py new file mode 100644 index 00000000000..e0894b825a7 --- /dev/null +++ b/litellm/litellm_core_utils/resource_ownership.py @@ -0,0 +1,93 @@ +from typing import Final + +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.types.proxy.auth.user_roles import LitellmUserRoles + + +def is_proxy_admin(user_api_key_dict: UserAPIKeyAuth | None) -> bool: + if user_api_key_dict is None: + return False + + return ( + user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN + or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value + ) + + +def get_resource_owner_scopes( + user_api_key_dict: UserAPIKeyAuth | None, +) -> list[str]: + """ + Return ownership scopes that may access a user-created proxy resource. + + Raw user_id is included for rows created before scope prefixes existed. + Prefixes avoid collisions when falling back to team/org/key ownership + for keys that do not have a user_id. + + Identity-less callers (no user_id, team_id, org_id, api_key, or token) + return ``[]`` — they share no scope with any other caller, so access + checks against an existing owner always fail and creates that depend + on a primary scope must reject up front. Returning a shared sentinel + here would let any two identity-less callers see each other's data. + """ + if user_api_key_dict is None: + return [] + + scopes: Final[list[str]] = [] + + def _add(scope: str | None) -> None: + if scope and scope not in scopes: + scopes.append(scope) + + if user_api_key_dict.user_id: + _add(user_api_key_dict.user_id) + _add(f"user:{user_api_key_dict.user_id}") + if user_api_key_dict.team_id: + _add(f"team:{user_api_key_dict.team_id}") + if user_api_key_dict.org_id: + _add(f"org:{user_api_key_dict.org_id}") + if user_api_key_dict.api_key: + _add(f"key:{user_api_key_dict.api_key}") + if user_api_key_dict.token: + _add(f"key:{user_api_key_dict.token}") + + return scopes + + +def get_primary_resource_owner_scope( + user_api_key_dict: UserAPIKeyAuth | None, +) -> str | None: + """Return the canonical owner scope to stamp on newly-created rows. + + ``None`` for identity-less callers — callers that depend on a primary + scope to record ownership must surface that as a hard error rather + than fall back to a shared sentinel (which would collapse every + identity-less caller into the same logical owner). + """ + if user_api_key_dict is None: + return None + + if user_api_key_dict.user_id: + return user_api_key_dict.user_id + if user_api_key_dict.team_id: + return f"team:{user_api_key_dict.team_id}" + if user_api_key_dict.org_id: + return f"org:{user_api_key_dict.org_id}" + if user_api_key_dict.api_key: + return f"key:{user_api_key_dict.api_key}" + if user_api_key_dict.token: + return f"key:{user_api_key_dict.token}" + return None + + +def user_can_access_resource_owner( + owner: str | None, + user_api_key_dict: UserAPIKeyAuth | None, +) -> bool: + if user_api_key_dict is None: + return True + if is_proxy_admin(user_api_key_dict): + return True + if owner is None: + return False + return owner in get_resource_owner_scopes(user_api_key_dict) diff --git a/litellm/llms/base_llm/managed_resources/isolation.py b/litellm/llms/base_llm/managed_resources/isolation.py index 6a71e8e9223..0a09de79891 100644 --- a/litellm/llms/base_llm/managed_resources/isolation.py +++ b/litellm/llms/base_llm/managed_resources/isolation.py @@ -29,7 +29,7 @@ def resolve_resource_owner_id( A key with neither a user_id nor a team_id would otherwise stamp ``created_by=None`` and be locked out of its own resources, so it owns them under its hashed token instead, using the ``key:`` scope prefix - already used by ``proxy/common_utils/resource_ownership.py``. ``None`` + already used by ``litellm_core_utils/resource_ownership.py``. ``None`` means the caller has no usable identity of its own and must fall back to team scoping, or be denied. """ diff --git a/litellm/llms/litellm_proxy/skills/handler.py b/litellm/llms/litellm_proxy/skills/handler.py index 0dbe47791fb..d56dca1699b 100644 --- a/litellm/llms/litellm_proxy/skills/handler.py +++ b/litellm/llms/litellm_proxy/skills/handler.py @@ -11,18 +11,18 @@ from typing import Final from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.resource_ownership import ( + get_primary_resource_owner_scope, + get_resource_owner_scopes, + is_proxy_admin, + user_can_access_resource_owner, +) from litellm.llms.litellm_proxy.skills.constants import ( LITELLM_SKILL_ID_PREFIX, MAX_SKILLS_PER_SEARCH, ) from litellm.models.skills import LiteLLM_SkillsTable, NewSkillRequest from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.common_utils.resource_ownership import ( - get_primary_resource_owner_scope, - get_resource_owner_scopes, - is_proxy_admin, - user_can_access_resource_owner, -) from litellm.repositories.table_repositories import SkillsRepository # Skills are looked up on every chat completion that has skills enabled diff --git a/litellm/proxy/common_utils/resource_ownership.py b/litellm/proxy/common_utils/resource_ownership.py index c75a5ba8e6e..985ad8a6f77 100644 --- a/litellm/proxy/common_utils/resource_ownership.py +++ b/litellm/proxy/common_utils/resource_ownership.py @@ -1,92 +1,20 @@ from typing import Final -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.litellm_core_utils.resource_ownership import ( + get_primary_resource_owner_scope, + get_resource_owner_scopes, + is_proxy_admin, + user_can_access_resource_owner, +) +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.types.proxy.auth.user_roles import LitellmUserRoles - -def is_proxy_admin(user_api_key_dict: UserAPIKeyAuth | None) -> bool: - if user_api_key_dict is None: - return False - - return ( - user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN - or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value - ) - - -def get_resource_owner_scopes( - user_api_key_dict: UserAPIKeyAuth | None, -) -> list[str]: - """ - Return ownership scopes that may access a user-created proxy resource. - - Raw user_id is included for rows created before scope prefixes existed. - Prefixes avoid collisions when falling back to team/org/key ownership - for keys that do not have a user_id. - - Identity-less callers (no user_id, team_id, org_id, api_key, or token) - return ``[]`` — they share no scope with any other caller, so access - checks against an existing owner always fail and creates that depend - on a primary scope must reject up front. Returning a shared sentinel - here would let any two identity-less callers see each other's data. - """ - if user_api_key_dict is None: - return [] - - scopes: Final[list[str]] = [] - - def _add(scope: str | None) -> None: - if scope and scope not in scopes: - scopes.append(scope) - - if user_api_key_dict.user_id: - _add(user_api_key_dict.user_id) - _add(f"user:{user_api_key_dict.user_id}") - if user_api_key_dict.team_id: - _add(f"team:{user_api_key_dict.team_id}") - if user_api_key_dict.org_id: - _add(f"org:{user_api_key_dict.org_id}") - if user_api_key_dict.api_key: - _add(f"key:{user_api_key_dict.api_key}") - if user_api_key_dict.token: - _add(f"key:{user_api_key_dict.token}") - - return scopes - - -def get_primary_resource_owner_scope( - user_api_key_dict: UserAPIKeyAuth | None, -) -> str | None: - """Return the canonical owner scope to stamp on newly-created rows. - - ``None`` for identity-less callers — callers that depend on a primary - scope to record ownership must surface that as a hard error rather - than fall back to a shared sentinel (which would collapse every - identity-less caller into the same logical owner). - """ - if user_api_key_dict is None: - return None - - if user_api_key_dict.user_id: - return user_api_key_dict.user_id - if user_api_key_dict.team_id: - return f"team:{user_api_key_dict.team_id}" - if user_api_key_dict.org_id: - return f"org:{user_api_key_dict.org_id}" - if user_api_key_dict.api_key: - return f"key:{user_api_key_dict.api_key}" - if user_api_key_dict.token: - return f"key:{user_api_key_dict.token}" - return None - - -def user_can_access_resource_owner( - owner: str | None, - user_api_key_dict: UserAPIKeyAuth | None, -) -> bool: - if user_api_key_dict is None: - return True - if is_proxy_admin(user_api_key_dict): - return True - if owner is None: - return False - return owner in get_resource_owner_scopes(user_api_key_dict) +__all__ = [ + "Final", + "LitellmUserRoles", + "UserAPIKeyAuth", + "get_primary_resource_owner_scope", + "get_resource_owner_scopes", + "is_proxy_admin", + "user_can_access_resource_owner", +] diff --git a/tests/unit/proxy/common_utils/test_resource_ownership.py b/tests/unit/proxy/common_utils/test_resource_ownership.py new file mode 100644 index 00000000000..a13f66dd59c --- /dev/null +++ b/tests/unit/proxy/common_utils/test_resource_ownership.py @@ -0,0 +1,29 @@ +from typing import Final + +from litellm.litellm_core_utils import resource_ownership as core_resource_ownership +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.common_utils import resource_ownership as proxy_resource_ownership + +PREVIOUSLY_EXPOSED_NAMES: Final = ( + "Final", + "LitellmUserRoles", + "UserAPIKeyAuth", + "get_primary_resource_owner_scope", + "get_resource_owner_scopes", + "is_proxy_admin", + "user_can_access_resource_owner", +) + + +def test_proxy_path_reexports_core_objects() -> None: + mismatched: Final = [ + name + for name in PREVIOUSLY_EXPOSED_NAMES + if getattr(proxy_resource_ownership, name) is not getattr(core_resource_ownership, name) + ] + assert mismatched == [] + + +def test_moved_module_uses_the_proxy_types_objects() -> None: + assert core_resource_ownership.LitellmUserRoles is LitellmUserRoles + assert core_resource_ownership.UserAPIKeyAuth is UserAPIKeyAuth From 5b6ecab52d16963c7d33047fd3976f94e43b8b1d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:03:33 -0700 Subject: [PATCH 20/40] fix(proxy): describe rejected characters in key_alias validation error (#41412) * fix(proxy): describe rejected characters in key_alias validation error Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): wrap key_alias validation error message Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Yuneng Jiang --- .../proxy/management_endpoints/key_management_endpoints.py | 5 ++++- .../management_endpoints/test_key_management_endpoints.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 6e226868918..782a7e1db3c 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -8040,7 +8040,10 @@ def _validate_key_alias_format(key_alias: str | None) -> None: raise_if_unsafe_secret_name(key_alias) except ValueError: raise ProxyException( - message="Invalid key_alias", + message=( + "Invalid key_alias: must not contain control characters (such as tab or newline) " + + 'or a ".." path segment' + ), type=ProxyErrorTypes.bad_request_error, param="key_alias", code=400, diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 308558450b1..ee39df92ef2 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -10733,6 +10733,7 @@ class TestValidateKeyAliasFormat: "foo\n- !grant\n role: !!admin\n member: attacker", "foo\rbar", "foo\x00bar", + "my-test-key\t", ], ) def test_validate_key_alias_format_rejects_traversal_and_control_chars_even_when_flag_disabled( @@ -10751,6 +10752,8 @@ class TestValidateKeyAliasFormat: _validate_key_alias_format(unsafe_alias) assert str(exc.value.code) == "400" assert "Invalid key_alias" in str(exc.value.message) + assert "control characters" in str(exc.value.message) + assert '".." path segment' in str(exc.value.message) def test_validate_key_alias_format_valid(self, monkeypatch): from litellm.proxy.management_endpoints.key_management_endpoints import ( From bc2cc42340a2f363c68011250dd545268d90dfdd Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 21:04:59 +0000 Subject: [PATCH 21/40] docs(proxy): describe /v1/decisions and /v1/systemone in Swagger UI (#45838) * docs(proxy): describe /v1/decisions and /v1/systemone in Swagger UI Add summaries, descriptions, inlined request body schemas with examples and typed 200 responses to the decision routes and their aliases, so the OpenAPI document and the lazy snapshot show both request formats Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): cap _inlined schema walk depth for the recursion check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_lazy_openapi_snapshot.json | 1805 ++++++++++++++++- .../proxy/common_utils/custom_openapi_spec.py | 38 + .../proxy/decisions_endpoints/endpoints.py | 89 +- litellm/types/decisions.py | 86 +- .../code_coverage_tests/recursive_detector.py | 1 + .../common_utils/test_custom_openapi_spec.py | 92 +- .../decisions_endpoints/test_endpoints.py | 64 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 866 +++++++- 8 files changed, 2961 insertions(+), 80 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 69db4525a17..534796032c6 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -9494,17 +9494,878 @@ }, "decisions": { "components": { - "schemas": {} + "schemas": { + "ChoiceAnswer": { + "additionalProperties": true, + "properties": { + "choice": { + "title": "Choice", + "type": "string" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "choice", + "confidence", + "probabilities" + ], + "title": "ChoiceAnswer", + "type": "object" + }, + "DecisionsResponse": { + "additionalProperties": true, + "properties": { + "answers": { + "additionalProperties": { + "discriminator": { + "mapping": { + "choice": "#/components/schemas/ChoiceAnswer", + "noul": "#/components/schemas/NoulAnswer", + "score": "#/components/schemas/ScoreAnswer" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/NoulAnswer" + }, + { + "$ref": "#/components/schemas/ChoiceAnswer" + }, + { + "$ref": "#/components/schemas/ScoreAnswer" + } + ] + }, + "title": "Answers", + "type": "object" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/components/schemas/DecisionsUsage" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "answers" + ], + "title": "DecisionsResponse", + "type": "object" + }, + "DecisionsUsage": { + "additionalProperties": true, + "properties": { + "input_tokens": { + "default": 0, + "title": "Input Tokens", + "type": "integer" + }, + "output_tokens": { + "default": 0, + "title": "Output Tokens", + "type": "integer" + } + }, + "title": "DecisionsUsage", + "type": "object" + }, + "NoulAnswer": { + "additionalProperties": true, + "properties": { + "noul": { + "title": "Noul", + "type": "number" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "noul" + ], + "title": "NoulAnswer", + "type": "object" + }, + "OpenAIChoiceAnswer": { + "properties": { + "choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Choice" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "probabilities": { + "items": { + "$ref": "#/components/schemas/OpenAIChoiceProbability" + }, + "title": "Probabilities", + "type": "array" + }, + "type": { + "const": "choice", + "default": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "choice", + "probabilities", + "confidence" + ], + "title": "OpenAIChoiceAnswer", + "type": "object" + }, + "OpenAIChoiceProbability": { + "properties": { + "probability": { + "title": "Probability", + "type": "number" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Value" + } + }, + "required": [ + "value", + "probability" + ], + "title": "OpenAIChoiceProbability", + "type": "object" + }, + "OpenAIDecisionInputTokensDetails": { + "properties": { + "cache_write_tokens": { + "default": 0, + "title": "Cache Write Tokens", + "type": "integer" + }, + "cached_tokens": { + "default": 0, + "title": "Cached Tokens", + "type": "integer" + } + }, + "title": "OpenAIDecisionInputTokensDetails", + "type": "object" + }, + "OpenAIDecisionOutputTokensDetails": { + "properties": { + "reasoning_tokens": { + "default": 0, + "title": "Reasoning Tokens", + "type": "integer" + } + }, + "title": "OpenAIDecisionOutputTokensDetails", + "type": "object" + }, + "OpenAIDecisionResponse": { + "additionalProperties": true, + "properties": { + "answers": { + "items": { + "discriminator": { + "mapping": { + "choice": "#/components/schemas/OpenAIChoiceAnswer", + "predicate": "#/components/schemas/OpenAIPredicateAnswer", + "refusal": "#/components/schemas/OpenAIRefusalAnswer", + "score": "#/components/schemas/OpenAIScoreAnswer" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/OpenAIPredicateAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIChoiceAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIScoreAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIRefusalAnswer" + } + ] + }, + "title": "Answers", + "type": "array" + }, + "model": { + "title": "Model", + "type": "string" + }, + "usage": { + "$ref": "#/components/schemas/OpenAIDecisionUsage" + } + }, + "required": [ + "model", + "answers", + "usage" + ], + "title": "OpenAIDecisionResponse", + "type": "object" + }, + "OpenAIDecisionUsage": { + "properties": { + "input_tokens": { + "title": "Input Tokens", + "type": "integer" + }, + "input_tokens_details": { + "$ref": "#/components/schemas/OpenAIDecisionInputTokensDetails", + "default": { + "cache_write_tokens": 0, + "cached_tokens": 0 + } + }, + "output_tokens": { + "title": "Output Tokens", + "type": "integer" + }, + "output_tokens_details": { + "$ref": "#/components/schemas/OpenAIDecisionOutputTokensDetails", + "default": { + "reasoning_tokens": 0 + } + }, + "total_tokens": { + "title": "Total Tokens", + "type": "integer" + } + }, + "required": [ + "input_tokens", + "output_tokens", + "total_tokens" + ], + "title": "OpenAIDecisionUsage", + "type": "object" + }, + "OpenAIPredicateAnswer": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "probability": { + "title": "Probability", + "type": "number" + }, + "type": { + "const": "predicate", + "default": "predicate", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "probability" + ], + "title": "OpenAIPredicateAnswer", + "type": "object" + }, + "OpenAIRefusalAnswer": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "type": { + "const": "refusal", + "default": "refusal", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name" + ], + "title": "OpenAIRefusalAnswer", + "type": "object" + }, + "OpenAIScoreAnswer": { + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "probabilities": { + "items": { + "$ref": "#/components/schemas/OpenAIScoreProbability" + }, + "title": "Probabilities", + "type": "array" + }, + "score": { + "title": "Score", + "type": "number" + }, + "type": { + "const": "score", + "default": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "score", + "probabilities", + "confidence" + ], + "title": "OpenAIScoreAnswer", + "type": "object" + }, + "OpenAIScoreProbability": { + "properties": { + "label": { + "title": "Label", + "type": "string" + }, + "probability": { + "title": "Probability", + "type": "number" + }, + "value": { + "title": "Value", + "type": "integer" + } + }, + "required": [ + "value", + "label", + "probability" + ], + "title": "OpenAIScoreProbability", + "type": "object" + }, + "ScoreAnswer": { + "additionalProperties": true, + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "legend": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "title": "Legend", + "type": "object" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "score": { + "title": "Score", + "type": "number" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "score", + "confidence", + "legend", + "probabilities" + ], + "title": "ScoreAnswer", + "type": "object" + } + } }, "paths": { "/decisions": { "post": { + "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these\nrules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_decisions_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "model": "luna", + "questions": [ + { + "instructions": "Is the customer asking for a refund?", + "name": "is_refund_request", + "type": "predicate" + }, + { + "choices": [ + { + "description": "can wait a week", + "value": "low" + }, + { + "description": "needs action today", + "value": "high" + } + ], + "instructions": "How urgent is this?", + "name": "urgency", + "type": "choice" + }, + { + "instructions": "How frustrated is the customer?", + "levels": [ + { + "label": "calm" + }, + { + "label": "mildly annoyed" + }, + { + "label": "angry" + } + ], + "name": "frustration", + "type": "score" + } + ] + }, + "schema": { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "text": { + "title": "Text", + "type": "string" + }, + "type": { + "const": "input_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "text" + ], + "title": "OpenAIDecisionInputText", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Detail" + }, + "image_url": { + "title": "Image Url", + "type": "string" + }, + "type": { + "const": "input_image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "image_url" + ], + "title": "OpenAIDecisionInputImage", + "type": "object" + } + ] + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "const": "user", + "default": "user", + "title": "Role", + "type": "string" + }, + "type": { + "const": "message", + "default": "message", + "title": "Type", + "type": "string" + } + }, + "required": [ + "content" + ], + "title": "OpenAIDecisionInputMessage", + "type": "object" + }, + "type": "array" + } + ], + "description": "The text being judged, or user messages whose content mixes input_text and input_image parts", + "title": "Input" + }, + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "description": "Predicate, choice and score questions. Answers come back in the same order", + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The yes/no question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "predicate", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "OpenAIPredicateQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "choices": { + "description": "The options the model picks from. Values must be unique", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Value" + } + }, + "required": [ + "value" + ], + "title": "OpenAIChoiceOption", + "type": "object" + }, + "maxItems": 255, + "minItems": 2, + "title": "Choices", + "type": "array" + }, + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "choices" + ], + "title": "OpenAIChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "levels": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "label": { + "title": "Label", + "type": "string" + } + }, + "required": [ + "label" + ], + "title": "OpenAIScoreLevel", + "type": "object" + }, + "maxItems": 10, + "minItems": 2, + "title": "Levels", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "levels" + ], + "title": "OpenAIScoreQuestion", + "type": "object" + } + ] + }, + "maxItems": 128, + "minItems": 1, + "title": "Questions", + "type": "array" + }, + "safety_identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "A stable id for your end user. Sent to OpenAI and dropped for other providers", + "title": "Safety Identifier" + } + }, + "required": [ + "input", + "questions", + "model" + ], + "title": "OpenAIDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/OpenAIDecisionResponse" + } } }, "description": "Successful Response" @@ -9515,7 +10376,7 @@ "APIKeyHeader": [] } ], - "summary": "Decisions", + "summary": "Ask questions about an input (OpenAI Decisions format)", "tags": [ "decisions" ] @@ -9523,12 +10384,290 @@ }, "/systemone": { "post": { + "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider\nis called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_systemone_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "jev", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "urgency": { + "criteria": { + "high": "needs action today", + "low": "can wait a week" + }, + "instructions": "How urgent is this?", + "type": "choice" + } + }, + "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + }, + "schema": { + "additionalProperties": true, + "properties": { + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": true, + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what makes the answer true and what makes it false", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The yes/no question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "NoulQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "description": "Candidate labels mapped to an optional description of each", + "maxProperties": 255, + "minProperties": 1, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "maxItems": 10, + "minItems": 1, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ScoreQuestion", + "type": "object" + } + ] + }, + "description": "Named noul, choice and score questions. Each key becomes a key in the answers", + "maxProperties": 128, + "minProperties": 1, + "propertyNames": { + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The thing being judged, such as a support ticket, a document or a chat transcript", + "title": "State" + } + }, + "required": [ + "state", + "questions", + "model" + ], + "title": "DecisionsRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/DecisionsResponse" + } } }, "description": "Successful Response" @@ -9539,7 +10678,7 @@ "APIKeyHeader": [] } ], - "summary": "Systemone", + "summary": "Ask named questions about a state (System One format)", "tags": [ "decisions" ] @@ -9547,12 +10686,382 @@ }, "/v1/decisions": { "post": { + "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these\nrules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_v1_decisions_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "model": "luna", + "questions": [ + { + "instructions": "Is the customer asking for a refund?", + "name": "is_refund_request", + "type": "predicate" + }, + { + "choices": [ + { + "description": "can wait a week", + "value": "low" + }, + { + "description": "needs action today", + "value": "high" + } + ], + "instructions": "How urgent is this?", + "name": "urgency", + "type": "choice" + }, + { + "instructions": "How frustrated is the customer?", + "levels": [ + { + "label": "calm" + }, + { + "label": "mildly annoyed" + }, + { + "label": "angry" + } + ], + "name": "frustration", + "type": "score" + } + ] + }, + "schema": { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "text": { + "title": "Text", + "type": "string" + }, + "type": { + "const": "input_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "text" + ], + "title": "OpenAIDecisionInputText", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Detail" + }, + "image_url": { + "title": "Image Url", + "type": "string" + }, + "type": { + "const": "input_image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "image_url" + ], + "title": "OpenAIDecisionInputImage", + "type": "object" + } + ] + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "const": "user", + "default": "user", + "title": "Role", + "type": "string" + }, + "type": { + "const": "message", + "default": "message", + "title": "Type", + "type": "string" + } + }, + "required": [ + "content" + ], + "title": "OpenAIDecisionInputMessage", + "type": "object" + }, + "type": "array" + } + ], + "description": "The text being judged, or user messages whose content mixes input_text and input_image parts", + "title": "Input" + }, + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "description": "Predicate, choice and score questions. Answers come back in the same order", + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The yes/no question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "predicate", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "OpenAIPredicateQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "choices": { + "description": "The options the model picks from. Values must be unique", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Value" + } + }, + "required": [ + "value" + ], + "title": "OpenAIChoiceOption", + "type": "object" + }, + "maxItems": 255, + "minItems": 2, + "title": "Choices", + "type": "array" + }, + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "choices" + ], + "title": "OpenAIChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "levels": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "label": { + "title": "Label", + "type": "string" + } + }, + "required": [ + "label" + ], + "title": "OpenAIScoreLevel", + "type": "object" + }, + "maxItems": 10, + "minItems": 2, + "title": "Levels", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "levels" + ], + "title": "OpenAIScoreQuestion", + "type": "object" + } + ] + }, + "maxItems": 128, + "minItems": 1, + "title": "Questions", + "type": "array" + }, + "safety_identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "A stable id for your end user. Sent to OpenAI and dropped for other providers", + "title": "Safety Identifier" + } + }, + "required": [ + "input", + "questions", + "model" + ], + "title": "OpenAIDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/OpenAIDecisionResponse" + } } }, "description": "Successful Response" @@ -9563,7 +11072,7 @@ "APIKeyHeader": [] } ], - "summary": "Decisions", + "summary": "Ask questions about an input (OpenAI Decisions format)", "tags": [ "decisions" ] @@ -9571,12 +11080,290 @@ }, "/v1/systemone": { "post": { + "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider\nis called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_v1_systemone_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "jev", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "urgency": { + "criteria": { + "high": "needs action today", + "low": "can wait a week" + }, + "instructions": "How urgent is this?", + "type": "choice" + } + }, + "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + }, + "schema": { + "additionalProperties": true, + "properties": { + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": true, + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what makes the answer true and what makes it false", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The yes/no question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "NoulQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "description": "Candidate labels mapped to an optional description of each", + "maxProperties": 255, + "minProperties": 1, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "maxItems": 10, + "minItems": 1, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ScoreQuestion", + "type": "object" + } + ] + }, + "description": "Named noul, choice and score questions. Each key becomes a key in the answers", + "maxProperties": 128, + "minProperties": 1, + "propertyNames": { + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The thing being judged, such as a support ticket, a document or a chat transcript", + "title": "State" + } + }, + "required": [ + "state", + "questions", + "model" + ], + "title": "DecisionsRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/DecisionsResponse" + } } }, "description": "Successful Response" @@ -9587,7 +11374,7 @@ "APIKeyHeader": [] } ], - "summary": "Systemone", + "summary": "Ask named questions about a state (System One format)", "tags": [ "decisions" ] diff --git a/litellm/proxy/common_utils/custom_openapi_spec.py b/litellm/proxy/common_utils/custom_openapi_spec.py index c1d4442893a..45ca509b7b7 100644 --- a/litellm/proxy/common_utils/custom_openapi_spec.py +++ b/litellm/proxy/common_utils/custom_openapi_spec.py @@ -437,3 +437,41 @@ class CustomOpenAPISpec: # Add responses API request schema return CustomOpenAPISpec.add_responses_api_request_schema(with_embeddings) + + +_MAX_INLINE_DEPTH: Final = 32 + + +def _inlined(node: JsonValue, defs: Mapping[str, JsonValue], depth: int = 0) -> JsonValue: + if depth >= _MAX_INLINE_DEPTH or not isinstance(node, (list, dict)): + return node + if isinstance(node, list): + return [_inlined(item, defs, depth + 1) for item in node] + ref: Final = node.get("$ref") + if isinstance(ref, str) and ref.startswith("#/$defs/"): + target: Final = _inlined(defs[ref.removeprefix("#/$defs/")], defs, depth + 1) + siblings: Final = {key: _inlined(value, defs, depth + 1) for key, value in node.items() if key != "$ref"} + return {**target, **siblings} if isinstance(target, dict) else siblings + if "propertyName" in node and "mapping" in node: + return {"propertyName": node["propertyName"]} + return {key: _inlined(value, defs, depth + 1) for key, value in node.items() if key != "$defs"} + + +def inline_request_body(model_class: type, example: Mapping[str, object]) -> JsonObject: + """ + Build an ``openapi_extra["requestBody"]`` for a route that reads its body with ``request.body()``. + + Swagger UI resolves ``$ref`` against the whole document, so a schema placed inline in an operation + cannot keep Pydantic's ``#/$defs/...`` references. Every reference is expanded in place instead, + which also keeps the operation self-contained inside the lazy OpenAPI snapshot. + """ + schema: Final = CustomOpenAPISpec.get_pydantic_schema(model_class) + media: Final[JsonObject] = {"example": cast(JsonValue, dict(example))} # cast-ok: JSON example literal + if schema is None: + return {"required": True, "content": {"application/json": media}} + raw_defs: Final = schema.get("$defs") + defs: Final[Mapping[str, JsonValue]] = raw_defs if isinstance(raw_defs, dict) else MappingProxyType({}) + return { + "required": True, + "content": {"application/json": {"schema": _inlined(schema, defs), **media}}, + } diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index 4f4209208ac..c8d4768dad0 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -9,7 +9,15 @@ from pydantic import TypeAdapter, ValidationError from litellm.exceptions import BadRequestError from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody +from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body +from litellm.types.decisions import ( + DecisionsRequest, + DecisionsRequestBody, + DecisionsResponse, + OpenAIDecisionRequest, + OpenAIDecisionRequestBody, + OpenAIDecisionResponse, +) router: Final = APIRouter() _REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) @@ -20,6 +28,51 @@ _OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBo _GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) _OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) +_SYSTEMONE_REQUEST_BODY: Final = inline_request_body( + DecisionsRequest, + { + "model": "jev", + "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "questions": { + "is_refund_request": {"type": "noul", "instructions": "Is the customer asking for a refund?"}, + "urgency": { + "type": "choice", + "instructions": "How urgent is this?", + "criteria": {"low": "can wait a week", "high": "needs action today"}, + }, + "frustration": { + "type": "score", + "instructions": "How frustrated is the customer?", + "criteria": ["calm", "mildly annoyed", "angry"], + }, + }, + }, +) +_DECISIONS_REQUEST_BODY: Final = inline_request_body( + OpenAIDecisionRequest, + { + "model": "luna", + "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "questions": [ + {"type": "predicate", "name": "is_refund_request", "instructions": "Is the customer asking for a refund?"}, + { + "type": "choice", + "name": "urgency", + "instructions": "How urgent is this?", + "choices": [ + {"value": "low", "description": "can wait a week"}, + {"value": "high", "description": "needs action today"}, + ], + }, + { + "type": "score", + "name": "frustration", + "instructions": "How frustrated is the customer?", + "levels": [{"label": "calm"}, {"label": "mildly annoyed"}, {"label": "angry"}], + }, + ], + }, +) async def _invalid_request( @@ -121,18 +174,35 @@ async def _process_decisions( dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask named questions about a state (System One format)", + responses={200: {"model": DecisionsResponse}}, + openapi_extra={"requestBody": _SYSTEMONE_REQUEST_BODY}, ) @router.post( "/systemone", dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask named questions about a state (System One format)", + responses={200: {"model": DecisionsResponse}}, + openapi_extra={"requestBody": _SYSTEMONE_REQUEST_BODY}, ) async def systemone( request: Request, fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): + """ + Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + + Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with + `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, + `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider + is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. + + [Docs](https://docs.litellm.ai/docs/decisions) + """ return await _process_decisions( request=request, fastapi_response=fastapi_response, @@ -146,18 +216,35 @@ async def systemone( dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask questions about an input (OpenAI Decisions format)", + responses={200: {"model": OpenAIDecisionResponse}}, + openapi_extra={"requestBody": _DECISIONS_REQUEST_BODY}, ) @router.post( "/decisions", dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask questions about an input (OpenAI Decisions format)", + responses={200: {"model": OpenAIDecisionResponse}}, + openapi_extra={"requestBody": _DECISIONS_REQUEST_BODY}, ) async def decisions( request: Request, fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): + """ + Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + + Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with + `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these + rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + model list. System One decision models accept this format too, LiteLLM translates the request and the + answers. Streaming is not supported. + + [Docs](https://docs.litellm.ai/docs/decisions) + """ return await _process_decisions( request=request, fastapi_response=fastapi_response, diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index b24690dbd9d..3bc90a5109a 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -15,8 +15,13 @@ MAX_DECISION_QUESTIONS: Final = 128 class NoulQuestion(LiteLLMPydanticObjectBase): type: Literal["noul"] - instructions: DecisionsJSON | None = None - criteria: NoulCriteria | None = None + instructions: Annotated[DecisionsJSON | None, Field(description="The yes/no question to ask about the state")] = ( + None + ) + criteria: Annotated[ + NoulCriteria | None, + Field(description="Optional descriptions of what makes the answer true and what makes it false"), + ] = None model_config = ConfigDict(extra="allow", frozen=True) @@ -29,16 +34,24 @@ class NoulQuestion(LiteLLMPydanticObjectBase): class ChoiceQuestion(LiteLLMPydanticObjectBase): type: Literal["choice"] - instructions: DecisionsJSON | None = None - criteria: Annotated[Mapping[str, DecisionsJSON | None], Field(min_length=1, max_length=255)] + instructions: Annotated[DecisionsJSON | None, Field(description="The question to ask about the state")] = None + criteria: Annotated[ + Mapping[str, DecisionsJSON | None], + Field(min_length=1, max_length=255, description="Candidate labels mapped to an optional description of each"), + ] model_config = ConfigDict(extra="allow", frozen=True) class ScoreQuestion(LiteLLMPydanticObjectBase): type: Literal["score"] - instructions: DecisionsJSON | None = None - criteria: Annotated[Sequence[DecisionsJSON], Field(min_length=1, max_length=10)] + instructions: Annotated[DecisionsJSON | None, Field(description="The question to ask about the state")] = None + criteria: Annotated[ + Sequence[DecisionsJSON], + Field( + min_length=1, max_length=10, description="Scale levels from lowest to highest. The score is a level index" + ), + ] model_config = ConfigDict(extra="allow", frozen=True) @@ -55,14 +68,20 @@ DecisionQuestionMap: TypeAlias = Annotated[ class DecisionsRequestBody(LiteLLMPydanticObjectBase): - state: DecisionsJSON - questions: DecisionQuestionMap + state: Annotated[ + DecisionsJSON, + Field(description="The thing being judged, such as a support ticket, a document or a chat transcript"), + ] + questions: Annotated[ + DecisionQuestionMap, + Field(description="Named noul, choice and score questions. Each key becomes a key in the answers"), + ] model_config = ConfigDict(extra="allow", frozen=True) class DecisionsRequest(DecisionsRequestBody): - model: str + model: Annotated[str, Field(description="A decision model from the proxy model_list")] @with_config(ConfigDict(extra="allow")) @@ -171,8 +190,8 @@ OpenAIDecisionInput: TypeAlias = str | Sequence[OpenAIDecisionInputMessage] class OpenAIPredicateQuestion(LiteLLMPydanticObjectBase): type: Literal["predicate"] - name: str | None = None - instructions: str + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="The yes/no question to ask about the input")] model_config = ConfigDict(extra="forbid", frozen=True) @@ -192,9 +211,12 @@ def systemone_choice_key(value: str | bool) -> str: class OpenAIChoiceQuestion(LiteLLMPydanticObjectBase): type: Literal["choice"] - name: str | None = None - instructions: str - choices: Annotated[Sequence[OpenAIChoiceOption], Field(min_length=2, max_length=255)] + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="The question to ask about the input")] + choices: Annotated[ + Sequence[OpenAIChoiceOption], + Field(min_length=2, max_length=255, description="The options the model picks from. Values must be unique"), + ] model_config = ConfigDict(extra="forbid", frozen=True) @@ -215,9 +237,14 @@ class OpenAIScoreLevel(LiteLLMPydanticObjectBase): class OpenAIScoreQuestion(LiteLLMPydanticObjectBase): type: Literal["score"] - name: str | None = None - instructions: str - levels: Annotated[Sequence[OpenAIScoreLevel], Field(min_length=2, max_length=10)] + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="The question to ask about the input")] + levels: Annotated[ + Sequence[OpenAIScoreLevel], + Field( + min_length=2, max_length=10, description="Scale levels from lowest to highest. The score is a level index" + ), + ] model_config = ConfigDict(extra="forbid", frozen=True) @@ -229,13 +256,32 @@ OpenAIDecisionQuestion: TypeAlias = Annotated[ class OpenAIDecisionRequestBody(LiteLLMPydanticObjectBase): - input: OpenAIDecisionInput - questions: Annotated[Sequence[OpenAIDecisionQuestion], Field(min_length=1, max_length=MAX_DECISION_QUESTIONS)] - safety_identifier: str | None = None + input: Annotated[ + OpenAIDecisionInput, + Field( + description="The text being judged, or user messages whose content mixes input_text and input_image parts" + ), + ] + questions: Annotated[ + Sequence[OpenAIDecisionQuestion], + Field( + min_length=1, + max_length=MAX_DECISION_QUESTIONS, + description="Predicate, choice and score questions. Answers come back in the same order", + ), + ] + safety_identifier: Annotated[ + str | None, + Field(description="A stable id for your end user. Sent to OpenAI and dropped for other providers"), + ] = None model_config = ConfigDict(extra="allow", frozen=True) +class OpenAIDecisionRequest(OpenAIDecisionRequestBody): + model: Annotated[str, Field(description="A decision model from the proxy model_list")] + + class OpenAIPredicateAnswer(LiteLLMPydanticObjectBase): type: Literal["predicate"] = "predicate" name: str | None diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index cea2b4154a7..e1d58ee73d2 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -58,6 +58,7 @@ IGNORE_FUNCTIONS = [ "sanitize_oci_schema", # OCI: bounded by JSON-schema tree depth. "_freeze_for_dedupe", # OTEL: max depth set (default 16, _FREEZE_MAX_DEPTH); fails closed by returning repr(value) at the cap. "apply_json_merge_patch", # max depth set (_MAX_MERGE_DEPTH=64); fails closed by raising ValueError at the cap. + "_inlined", # max depth set (_MAX_INLINE_DEPTH=32); passes the schema node through untouched at the cap. Walks a Pydantic JSON schema once at import time. "_filter_argument_value", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by blocking the tool call at the cap. "_redact_scanned_content", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by returning "[REDACTED]" at the cap. "replace_ciphertexts", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); walks stored JSON, which has no cycles, and leaves values below the cap untouched. diff --git a/tests/unit/proxy/common_utils/test_custom_openapi_spec.py b/tests/unit/proxy/common_utils/test_custom_openapi_spec.py index 485dbcef2f6..41ab613e7ea 100644 --- a/tests/unit/proxy/common_utils/test_custom_openapi_spec.py +++ b/tests/unit/proxy/common_utils/test_custom_openapi_spec.py @@ -4,11 +4,14 @@ Simple unit tests for CustomOpenAPISpec class. Tests basic functionality of OpenAPI schema generation. """ -from unittest.mock import Mock, patch +import json +from typing import Annotated, Literal +from unittest.mock import patch import pytest +from pydantic import BaseModel, ConfigDict, Field -from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec +from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec, inline_request_body class TestCustomOpenAPISpec: @@ -27,19 +30,13 @@ class TestCustomOpenAPISpec: }, } - @patch( - "litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema" - ) - def test_add_chat_completion_request_schema( - self, mock_add_schema, base_openapi_schema - ): + @patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema") + def test_add_chat_completion_request_schema(self, mock_add_schema, base_openapi_schema): """Test that chat completion schema is added correctly.""" mock_add_schema.return_value = base_openapi_schema with patch("litellm.proxy._types.ProxyChatCompletionRequest") as mock_model: - result = CustomOpenAPISpec.add_chat_completion_request_schema( - base_openapi_schema - ) + result = CustomOpenAPISpec.add_chat_completion_request_schema(base_openapi_schema) mock_add_schema.assert_called_once_with( openapi_schema=base_openapi_schema, @@ -50,9 +47,7 @@ class TestCustomOpenAPISpec: ) assert result == base_openapi_schema - @patch( - "litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema" - ) + @patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema") def test_add_embedding_request_schema(self, mock_add_schema, base_openapi_schema): """Test that embedding schema is added correctly.""" mock_add_schema.return_value = base_openapi_schema @@ -69,19 +64,13 @@ class TestCustomOpenAPISpec: ) assert result == base_openapi_schema - @patch( - "litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema" - ) - def test_add_responses_api_request_schema( - self, mock_add_schema, base_openapi_schema - ): + @patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema") + def test_add_responses_api_request_schema(self, mock_add_schema, base_openapi_schema): """Test that responses API schema is added correctly.""" mock_add_schema.return_value = base_openapi_schema with patch("litellm.types.llms.openai.ResponsesAPIRequestParams") as mock_model: - result = CustomOpenAPISpec.add_responses_api_request_schema( - base_openapi_schema - ) + result = CustomOpenAPISpec.add_responses_api_request_schema(base_openapi_schema) mock_add_schema.assert_called_once_with( openapi_schema=base_openapi_schema, @@ -123,15 +112,11 @@ def test_defs_rewritten_in_add_schema_to_components(): ) assert "$defs" not in openapi_schema assert ( - openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"][ - "items" - ]["anyOf"][0]["$ref"] + openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"]["items"]["anyOf"][0]["$ref"] == "#/components/schemas/UserMessage" ) assert ( - openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"][ - "items" - ]["anyOf"][1]["$ref"] + openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"]["items"]["anyOf"][1]["$ref"] == "#/components/schemas/AssistantMessage" ) @@ -188,14 +173,8 @@ def test_rewrite_defs_refs(): rewritten = CustomOpenAPISpec._rewrite_defs_refs(schema=schema, renames={}) assert "$defs" not in rewritten - assert ( - rewritten["properties"]["messages"]["items"]["anyOf"][0]["$ref"] - == "#/components/schemas/UserMessage" - ) - assert ( - rewritten["properties"]["messages"]["items"]["anyOf"][1]["$ref"] - == "#/components/schemas/AssistantMessage" - ) + assert rewritten["properties"]["messages"]["items"]["anyOf"][0]["$ref"] == "#/components/schemas/UserMessage" + assert rewritten["properties"]["messages"]["items"]["anyOf"][1]["$ref"] == "#/components/schemas/AssistantMessage" def test_get_pydantic_schema_generates_schema_for_responses_request_typed_dict(): @@ -390,3 +369,42 @@ def test_add_schema_to_components_renames_def_with_different_required_set(): assert schemas["Block"]["required"] == ["type", "x", "y"] assert schemas["Req_Block"]["required"] == ["keys", "type", "x", "y"] assert schemas["Req"]["properties"]["b"]["$ref"] == "#/components/schemas/Req_Block" + + +class _Cat(BaseModel): + kind: Literal["cat"] + lives: int = 9 + + model_config = ConfigDict(frozen=True) + + +class _Dog(BaseModel): + kind: Literal["dog"] + good: bool = True + + model_config = ConfigDict(frozen=True) + + +class _Shelter(BaseModel): + pets: dict[str, Annotated[_Cat | _Dog, Field(discriminator="kind")]] + favorite: _Cat | None = None + + model_config = ConfigDict(frozen=True) + + +def test_inline_request_body_expands_every_ref_so_the_operation_stands_alone(): + body = inline_request_body(_Shelter, {"pets": {"tom": {"kind": "cat"}}}) + + media = body["content"]["application/json"] + assert body["required"] is True + assert media["example"] == {"pets": {"tom": {"kind": "cat"}}} + assert "$ref" not in json.dumps(media["schema"]) + assert "$defs" not in media["schema"] + pet_schema = media["schema"]["properties"]["pets"]["additionalProperties"] + variants = {variant["properties"]["kind"]["const"]: variant for variant in pet_schema["oneOf"]} + assert variants["cat"]["properties"]["lives"]["default"] == 9 + assert variants["dog"]["properties"]["good"]["type"] == "boolean" + assert pet_schema["discriminator"] == {"propertyName": "kind"} + favorite = media["schema"]["properties"]["favorite"] + assert favorite["default"] is None + assert any(option.get("properties", {}).get("lives") for option in favorite["anyOf"]) diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index ed78fc7063a..87b2b64ad8d 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -12,17 +12,19 @@ import respx from fastapi import FastAPI from fastapi.routing import APIRoute from fastapi.testclient import TestClient +from pydantic import TypeAdapter from starlette.routing import Match import litellm from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature, attach_lazy_features -from litellm.proxy.decisions_endpoints.endpoints import decisions, systemone +from litellm.proxy.decisions_endpoints.endpoints import decisions, router, systemone from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder from litellm.proxy.proxy_server import ( app, cleanup_router_config_variables, initialize, ) +from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody _INPUT_TOKENS: Final[int] = 367 _OUTPUT_TOKENS: Final[int] = 3 @@ -688,3 +690,63 @@ def test_with_lazy_routes_disabled_a_config_pass_through_at_v1_decisions_still_w assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"} assert _serving_endpoint(bare, "/v1/decisions") is pass_through assert _serving_endpoint(bare, "/decisions") is decisions + + +@pytest.fixture(scope="module") +def decisions_openapi() -> dict[str, object]: + app = FastAPI() + app.include_router(router) + return app.openapi() + + +@pytest.mark.parametrize( + ("path", "body_adapter", "response_schema", "required"), + ( + ("/v1/systemone", TypeAdapter(DecisionsRequestBody), "DecisionsResponse", {"model", "state", "questions"}), + ("/systemone", TypeAdapter(DecisionsRequestBody), "DecisionsResponse", {"model", "state", "questions"}), + ( + "/v1/decisions", + TypeAdapter(OpenAIDecisionRequestBody), + "OpenAIDecisionResponse", + {"model", "input", "questions"}, + ), + ( + "/decisions", + TypeAdapter(OpenAIDecisionRequestBody), + "OpenAIDecisionResponse", + {"model", "input", "questions"}, + ), + ), +) +def test_swagger_documents_the_request_format_and_response_of_each_decisions_route( + decisions_openapi: dict[str, object], + path: str, + body_adapter: TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody], + response_schema: str, + required: set[str], +) -> None: + operation = decisions_openapi["paths"][path]["post"] + media = operation["requestBody"]["content"]["application/json"] + + assert "Decisions" in operation["summary"] or "System One" in operation["summary"] + assert "docs.litellm.ai/docs/decisions" in operation["description"] + assert required <= set(media["schema"]["required"]) + assert required <= set(media["schema"]["properties"]) + assert "$ref" not in json.dumps(media["schema"]) + assert media["schema"]["properties"]["model"]["description"] + + example = {key: value for key, value in media["example"].items() if key != "model"} + assert len(body_adapter.validate_python(example).questions) == 3 + + response_ref = operation["responses"]["200"]["content"]["application/json"]["schema"]["$ref"] + assert response_ref == f"#/components/schemas/{response_schema}" + assert "answers" in decisions_openapi["components"]["schemas"][response_schema]["properties"] + + +def test_the_two_decisions_routes_document_different_request_formats(decisions_openapi: dict[str, object]) -> None: + def properties(path: str) -> set[str]: + operation = decisions_openapi["paths"][path]["post"] + return set(operation["requestBody"]["content"]["application/json"]["schema"]["properties"]) + + assert "state" in properties("/v1/systemone") and "state" not in properties("/v1/decisions") + assert "input" in properties("/v1/decisions") and "input" not in properties("/v1/systemone") diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 47df4593a65..05e80d26e09 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -4703,7 +4703,18 @@ export interface paths { }; get?: never; put?: never; - /** Decisions */ + /** + * Ask questions about an input (OpenAI Decisions format) + * @description Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + * + * Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with + * `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these + * rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + * model list. System One decision models accept this format too, LiteLLM translates the request and the + * answers. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["decisions_decisions_post"]; delete?: never; options?: never; @@ -15837,7 +15848,18 @@ export interface paths { }; get?: never; put?: never; - /** Systemone */ + /** + * Ask named questions about a state (System One format) + * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + * + * Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with + * `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, + * `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider + * is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + * too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["systemone_systemone_post"]; delete?: never; options?: never; @@ -19646,7 +19668,18 @@ export interface paths { }; get?: never; put?: never; - /** Decisions */ + /** + * Ask questions about an input (OpenAI Decisions format) + * @description Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + * + * Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with + * `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these + * rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + * model list. System One decision models accept this format too, LiteLLM translates the request and the + * answers. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["decisions_v1_decisions_post"]; delete?: never; options?: never; @@ -21991,7 +22024,18 @@ export interface paths { }; get?: never; put?: never; - /** Systemone */ + /** + * Ask named questions about a state (System One format) + * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + * + * Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with + * `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, + * `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider + * is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + * too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["systemone_v1_systemone_post"]; delete?: never; options?: never; @@ -28642,6 +28686,24 @@ export interface components { */ role: "user" | "assistant"; }; + /** ChoiceAnswer */ + ChoiceAnswer: { + /** Choice */ + choice: string; + /** Confidence */ + confidence: number; + /** Probabilities */ + probabilities: { + [key: string]: number; + }; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "choice"; + } & { + [key: string]: unknown; + }; /** ChoiceLogprobs */ ChoiceLogprobs: { /** Content */ @@ -31008,6 +31070,33 @@ export interface components { */ total_tokens: number; }; + /** DecisionsResponse */ + DecisionsResponse: { + /** Answers */ + answers: { + [key: string]: components["schemas"]["NoulAnswer"] | components["schemas"]["ChoiceAnswer"] | components["schemas"]["ScoreAnswer"]; + }; + /** Model */ + model?: string | null; + usage?: components["schemas"]["DecisionsUsage"] | null; + } & { + [key: string]: unknown; + }; + /** DecisionsUsage */ + DecisionsUsage: { + /** + * Input Tokens + * @default 0 + */ + input_tokens: number; + /** + * Output Tokens + * @default 0 + */ + output_tokens: number; + } & { + [key: string]: unknown; + }; /** * DefaultInternalUserParams * @description Default parameters to apply when a new user signs in via SSO or is created on the /user/new API endpoint @@ -38834,6 +38923,18 @@ export interface components { /** User Role */ user_role?: ("proxy_admin" | "proxy_admin_viewer" | "internal_user" | "internal_user_viewer") | null; }; + /** NoulAnswer */ + NoulAnswer: { + /** Noul */ + noul: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "noul"; + } & { + [key: string]: unknown; + }; /** * OAuth2SecurityScheme * @description Defines a security scheme using OAuth 2.0. @@ -38872,6 +38973,129 @@ export interface components { [key: string]: unknown; } | null; }; + /** OpenAIChoiceAnswer */ + OpenAIChoiceAnswer: { + /** Choice */ + choice: string | boolean; + /** Confidence */ + confidence: number; + /** Name */ + name: string | null; + /** Probabilities */ + probabilities: components["schemas"]["OpenAIChoiceProbability"][]; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "choice"; + }; + /** OpenAIChoiceProbability */ + OpenAIChoiceProbability: { + /** Probability */ + probability: number; + /** Value */ + value: string | boolean; + }; + /** OpenAIDecisionInputTokensDetails */ + OpenAIDecisionInputTokensDetails: { + /** + * Cache Write Tokens + * @default 0 + */ + cache_write_tokens: number; + /** + * Cached Tokens + * @default 0 + */ + cached_tokens: number; + }; + /** OpenAIDecisionOutputTokensDetails */ + OpenAIDecisionOutputTokensDetails: { + /** + * Reasoning Tokens + * @default 0 + */ + reasoning_tokens: number; + }; + /** OpenAIDecisionResponse */ + OpenAIDecisionResponse: { + /** Answers */ + answers: (components["schemas"]["OpenAIPredicateAnswer"] | components["schemas"]["OpenAIChoiceAnswer"] | components["schemas"]["OpenAIScoreAnswer"] | components["schemas"]["OpenAIRefusalAnswer"])[]; + /** Model */ + model: string; + usage: components["schemas"]["OpenAIDecisionUsage"]; + } & { + [key: string]: unknown; + }; + /** OpenAIDecisionUsage */ + OpenAIDecisionUsage: { + /** Input Tokens */ + input_tokens: number; + /** + * @default { + * "cache_write_tokens": 0, + * "cached_tokens": 0 + * } + */ + input_tokens_details: components["schemas"]["OpenAIDecisionInputTokensDetails"]; + /** Output Tokens */ + output_tokens: number; + /** + * @default { + * "reasoning_tokens": 0 + * } + */ + output_tokens_details: components["schemas"]["OpenAIDecisionOutputTokensDetails"]; + /** Total Tokens */ + total_tokens: number; + }; + /** OpenAIPredicateAnswer */ + OpenAIPredicateAnswer: { + /** Name */ + name: string | null; + /** Probability */ + probability: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "predicate"; + }; + /** OpenAIRefusalAnswer */ + OpenAIRefusalAnswer: { + /** Name */ + name: string | null; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "refusal"; + }; + /** OpenAIScoreAnswer */ + OpenAIScoreAnswer: { + /** Confidence */ + confidence: number; + /** Name */ + name: string | null; + /** Probabilities */ + probabilities: components["schemas"]["OpenAIScoreProbability"][]; + /** Score */ + score: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "score"; + }; + /** OpenAIScoreProbability */ + OpenAIScoreProbability: { + /** Label */ + label: string; + /** Probability */ + probability: number; + /** Value */ + value: number; + }; /** * OpenIdConnectSecurityScheme * @description Defines a security scheme using OpenID Connect. @@ -43986,6 +44210,30 @@ export interface components { */ window_seconds: number; }; + /** ScoreAnswer */ + ScoreAnswer: { + /** Confidence */ + confidence: number; + /** Legend */ + legend: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[]; + }; + /** Probabilities */ + probabilities: { + [key: string]: number; + }; + /** Score */ + score: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "score"; + } & { + [key: string]: unknown; + }; /** * Screenshot * @description A screenshot action. @@ -56556,7 +56804,179 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + * "model": "luna", + * "questions": [ + * { + * "instructions": "Is the customer asking for a refund?", + * "name": "is_refund_request", + * "type": "predicate" + * }, + * { + * "choices": [ + * { + * "description": "can wait a week", + * "value": "low" + * }, + * { + * "description": "needs action today", + * "value": "high" + * } + * ], + * "instructions": "How urgent is this?", + * "name": "urgency", + * "type": "choice" + * }, + * { + * "instructions": "How frustrated is the customer?", + * "levels": [ + * { + * "label": "calm" + * }, + * { + * "label": "mildly annoyed" + * }, + * { + * "label": "angry" + * } + * ], + * "name": "frustration", + * "type": "score" + * } + * ] + * } + */ + "application/json": { + /** + * Input + * @description The text being judged, or user messages whose content mixes input_text and input_image parts + */ + input: string | { + /** Content */ + content: string | ({ + /** Text */ + text: string; + /** + * Type + * @constant + */ + type: "input_text"; + } | { + /** Detail */ + detail?: string | null; + /** Image Url */ + image_url: string; + /** + * Type + * @constant + */ + type: "input_image"; + })[]; + /** + * Role + * @default user + * @constant + */ + role?: "user"; + /** + * Type + * @default message + * @constant + */ + type?: "message"; + }[]; + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Predicate, choice and score questions. Answers come back in the same order + */ + questions: ({ + /** + * Instructions + * @description The yes/no question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "predicate"; + } | { + /** + * Choices + * @description The options the model picks from. Values must be unique + */ + choices: { + /** Description */ + description?: string | null; + /** Value */ + value: string | boolean; + }[]; + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Levels + * @description Scale levels from lowest to highest. The score is a level index + */ + levels: { + /** Description */ + description?: string | null; + /** Label */ + label: string; + }[]; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "score"; + })[]; + /** + * Safety Identifier + * @description A stable id for your end user. Sent to OpenAI and dropped for other providers + */ + safety_identifier?: string | null; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -56564,7 +56984,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["OpenAIDecisionResponse"]; }; }; }; @@ -70673,7 +71093,132 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "jev", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "urgency": { + * "criteria": { + * "high": "needs action today", + * "low": "can wait a week" + * }, + * "instructions": "How urgent is this?", + * "type": "choice" + * } + * }, + * "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + * } + */ + "application/json": { + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Named noul, choice and score questions. Each key becomes a key in the answers + */ + questions: { + [key: string]: ({ + /** + * Criteria + * @description Optional descriptions of what makes the answer true and what makes it false + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description The yes/no question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Candidate labels mapped to an optional description of each + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Scale levels from lowest to highest. The score is a level index + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[])[]; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + } & { + [key: string]: unknown; + }); + }; + /** + * State + * @description The thing being judged, such as a support ticket, a document or a chat transcript + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -70681,7 +71226,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["DecisionsResponse"]; }; }; }; @@ -75840,7 +76385,179 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + * "model": "luna", + * "questions": [ + * { + * "instructions": "Is the customer asking for a refund?", + * "name": "is_refund_request", + * "type": "predicate" + * }, + * { + * "choices": [ + * { + * "description": "can wait a week", + * "value": "low" + * }, + * { + * "description": "needs action today", + * "value": "high" + * } + * ], + * "instructions": "How urgent is this?", + * "name": "urgency", + * "type": "choice" + * }, + * { + * "instructions": "How frustrated is the customer?", + * "levels": [ + * { + * "label": "calm" + * }, + * { + * "label": "mildly annoyed" + * }, + * { + * "label": "angry" + * } + * ], + * "name": "frustration", + * "type": "score" + * } + * ] + * } + */ + "application/json": { + /** + * Input + * @description The text being judged, or user messages whose content mixes input_text and input_image parts + */ + input: string | { + /** Content */ + content: string | ({ + /** Text */ + text: string; + /** + * Type + * @constant + */ + type: "input_text"; + } | { + /** Detail */ + detail?: string | null; + /** Image Url */ + image_url: string; + /** + * Type + * @constant + */ + type: "input_image"; + })[]; + /** + * Role + * @default user + * @constant + */ + role?: "user"; + /** + * Type + * @default message + * @constant + */ + type?: "message"; + }[]; + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Predicate, choice and score questions. Answers come back in the same order + */ + questions: ({ + /** + * Instructions + * @description The yes/no question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "predicate"; + } | { + /** + * Choices + * @description The options the model picks from. Values must be unique + */ + choices: { + /** Description */ + description?: string | null; + /** Value */ + value: string | boolean; + }[]; + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Levels + * @description Scale levels from lowest to highest. The score is a level index + */ + levels: { + /** Description */ + description?: string | null; + /** Label */ + label: string; + }[]; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "score"; + })[]; + /** + * Safety Identifier + * @description A stable id for your end user. Sent to OpenAI and dropped for other providers + */ + safety_identifier?: string | null; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -75848,7 +76565,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["OpenAIDecisionResponse"]; }; }; }; @@ -79013,7 +79730,132 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "jev", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "urgency": { + * "criteria": { + * "high": "needs action today", + * "low": "can wait a week" + * }, + * "instructions": "How urgent is this?", + * "type": "choice" + * } + * }, + * "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + * } + */ + "application/json": { + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Named noul, choice and score questions. Each key becomes a key in the answers + */ + questions: { + [key: string]: ({ + /** + * Criteria + * @description Optional descriptions of what makes the answer true and what makes it false + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description The yes/no question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Candidate labels mapped to an optional description of each + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Scale levels from lowest to highest. The score is a level index + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[])[]; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + } & { + [key: string]: unknown; + }); + }; + /** + * State + * @description The thing being judged, such as a support ticket, a document or a chat transcript + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -79021,7 +79863,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["DecisionsResponse"]; }; }; }; From dc7738195639c92dc01a1b5ed5ede96be980083f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:05:03 -0700 Subject: [PATCH 22/40] feat(ui): replace Reset example with an Example preset dropdown and a Clear button on the Decisions tab (#45848) * feat(ui): replace Reset example with an Example preset dropdown and a Clear button on the Decisions tab Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): match Decisions presets by content and keep an omitted model out of Clear and preset picks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../SystemOneUI.integration.test.tsx | 87 ++++++++- .../components/systemOneUI/SystemOneUI.tsx | 58 ++++-- .../systemOneUI/lib/example.test.ts | 60 ++++++ .../components/systemOneUI/lib/example.ts | 180 +++++++++++++++--- 4 files changed, 340 insertions(+), 45 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/example.test.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx index ab0a67dbbab..c5c6023cf27 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx @@ -71,6 +71,12 @@ const selectEndpoint = async (endpoint: "/v1/decisions" | "/v1/systemone") => { await userEvent.click(await screen.findByRole("option", { name: endpoint, exact: true })); }; +const selectPreset = async (label: string) => { + screen.getByRole("combobox", { name: "Example" }).focus(); + await userEvent.keyboard("{ArrowDown}"); + await userEvent.click(await screen.findByRole("option", { name: label })); +}; + const createResponse = (body: SystemOneResponse | OpenAIDecisionsResponse, status = 200, errorText = "") => ({ ok: status >= 200 && status < 300, @@ -99,20 +105,81 @@ describe("SystemOneUI integration", () => { document.cookie = "token=; path=/; max-age=0"; }); - it("restores the example after the request is edited", async () => { - const user = userEvent.setup(); + it("switches between example presets and reads Custom once the request is edited", async () => { render(); const editor = await openJsonEditor(); const example = (editor as HTMLTextAreaElement).value; - const resetButton = screen.getByRole("button", { name: "Reset example" }); - expect(resetButton).toBeDisabled(); + const picker = screen.getByRole("combobox", { name: "Example" }); + expect(picker).toHaveTextContent("Triage a bug report"); fireEvent.change(editor, { target: { value: "{}" } }); - expect(resetButton).toBeEnabled(); - await user.click(resetButton); - + expect(picker).toHaveTextContent("Custom"); + await selectPreset("Triage a bug report"); expect(editor).toHaveValue(example); - expect(resetButton).toBeDisabled(); + expect(picker).toHaveTextContent("Triage a bug report"); + + await selectPreset("Score a sales lead"); + expect(picker).toHaveTextContent("Score a sales lead"); + expect(JSON.parse((editor as HTMLTextAreaElement).value)).toMatchObject({ questions: { fit: { type: "score" } } }); + }); + + it("clears the whole request and lets a preset be picked from nothing", async () => { + const user = userEvent.setup(); + render(); + const clear = screen.getByRole("button", { name: "Clear" }); + expect(screen.getByRole("textbox", { name: "Input" })).toHaveDisplayValue(/streaming responses/); + + await user.click(clear); + expect(screen.getByRole("textbox", { name: "Input" })).toHaveValue(""); + expect(screen.queryAllByRole("group", { name: /Question / })).toHaveLength(0); + expect(screen.getByRole("button", { name: "Send" })).toBeDisabled(); + expect(clear).toBeDisabled(); + expect(screen.getByRole("combobox", { name: "Example" })).toHaveTextContent("Custom"); + + await selectPreset("Route a support ticket"); + expect(screen.getByRole("textbox", { name: "Input" })).toHaveDisplayValue(/charged twice/); + expect(screen.getAllByRole("group", { name: /Question / })).toHaveLength(1); + expect(screen.getByRole("button", { name: "Send" })).toBeEnabled(); + }); + + it("keeps the preset label after picking a model and leaves an omitted model out when clearing", async () => { + mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS)); + render(); + await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest")); + const picker = screen.getByRole("combobox", { name: "Example" }); + const editor = await openJsonEditor(); + const example = JSON.parse((editor as HTMLTextAreaElement).value) as Record; + const { model: _model, ...withoutModel } = example; + + fireEvent.change(editor, { + target: { value: JSON.stringify({ ...withoutModel, model: "pplx-decider" }, null, 2) }, + }); + expect(picker).toHaveTextContent("Triage a bug report"); + + fireEvent.change(editor, { target: { value: JSON.stringify(withoutModel, null, 2) } }); + expect(picker).toHaveTextContent("Triage a bug report"); + await userEvent.click(screen.getByRole("button", { name: "Clear" })); + expect(JSON.parse((editor as HTMLTextAreaElement).value)).toEqual({ state: "", questions: {} }); + await selectPreset("Score a sales lead"); + expect(JSON.parse((editor as HTMLTextAreaElement).value)).not.toHaveProperty("model"); + }); + + it("offers the same presets on /v1/decisions in the OpenAI shape and clears to an empty input", async () => { + const user = userEvent.setup(); + render(); + await selectEndpoint("/v1/decisions"); + await selectPreset("Moderate a forum comment"); + await user.click(screen.getByRole("tab", { name: "JSON" })); + const editor = screen.getByRole("textbox", { name: "Decisions JSON payload" }) as HTMLTextAreaElement; + const payload = JSON.parse(editor.value); + expect(payload.input).toMatch(/stream=false/); + expect(payload.questions).toEqual([ + { type: "predicate", name: "breaks_rules", instructions: "Does this comment break the community guidelines?" }, + ]); + + await user.click(screen.getByRole("button", { name: "Clear" })); + expect(JSON.parse(editor.value)).toEqual({ input: "", questions: [] }); + expect(screen.getByRole("button", { name: "Send" })).toBeDisabled(); }); it("offers both literal proxy endpoints without TypeSafe or product prefixes", async () => { @@ -227,7 +294,7 @@ describe("SystemOneUI integration", () => { await selectEndpoint("/v1/decisions"); expect(screen.getByRole("textbox", { name: "Decisions JSON payload" })).toHaveValue(JSON.stringify(payload)); - await user.click(screen.getByRole("button", { name: "Reset example" })); + await selectPreset("Triage a bug report"); expect(screen.getByRole("textbox", { name: "Decisions JSON payload" })).toHaveValue(nativeExample); await selectEndpoint("/v1/systemone"); expect(screen.getByRole("textbox", { name: "System One JSON payload" })).toHaveValue(systemOneDraft); @@ -403,7 +470,7 @@ describe("SystemOneUI integration", () => { ); expect(next).toEqual({ ...payload, questions: [{ ...payload.questions[0], instructions: "Is it urgent?" }] }); await user.click(screen.getByRole("tab", { name: "Form" })); - await user.click(screen.getByRole("button", { name: "Reset example" })); + await selectPreset("Triage a bug report"); expect(screen.getByRole("textbox", { name: "Input" })).toBeInTheDocument(); expect(screen.getAllByRole("group", { name: /Question \d/ })).toHaveLength(3); await selectEndpoint("/v1/systemone"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx index 02d42851b8e..c46238315ff 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx @@ -9,10 +9,10 @@ import { SearchSelect } from "@/components/shared/SearchSelect"; import { DECISIONS_DOCS_URL } from "@/lib/decisionModels"; import { uiHref } from "@/utils/uiHref"; import { useMutation } from "@tanstack/react-query"; -import { Code, Info, LoaderCircle, RotateCcw, Send } from "lucide-react"; +import { Code, Eraser, Info, LoaderCircle, Send } from "lucide-react"; import { useEffect, useMemo, useRef, useState } from "react"; import { makePlaygroundDecisionRequest, type PlaygroundDecisionRequest } from "../../llm_calls/system_one"; -import { decisionsExample, openAIDecisionsExample } from "./lib/example"; +import { DECISION_PRESETS, emptyPayload, matchPreset, presetPayload, type DecisionPreset } from "./lib/example"; import { payloadModel, withPayloadModel } from "./lib/payloadModel"; import JsonEditor from "./JsonEditor"; import QuestionBreakdown from "./QuestionBreakdown"; @@ -35,8 +35,36 @@ interface SystemOneSendVariables { signal: AbortSignal; } +const CUSTOM_PRESET = "custom"; const DECISION_MODELS_DISCUSSION_URL = "https://github.com/BerriAI/litellm/discussions/44231"; +interface PresetPickerProps { + activePreset: DecisionPreset | undefined; + onPick: (id: string | null) => void; +} + +function PresetPicker({ activePreset, onPick }: PresetPickerProps) { + return ( + + ); +} + function getCustomProxyBaseUrl(): string | undefined { return typeof window === "undefined" ? undefined : window.sessionStorage.getItem("customProxyBaseUrl") || undefined; } @@ -53,9 +81,10 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = const [endpoint, setEndpoint] = useState("/v1/systemone"); const [drafts, setDrafts] = useState>>({}); const [view, setView] = useState("form"); - const example = endpoint === "/v1/decisions" ? openAIDecisionsExample : decisionsExample; - const examplePayload = JSON.stringify(example(decisionModels[0]), null, 2); - const rawPayload = drafts[endpoint] ?? examplePayload; + const rawPayload = drafts[endpoint] ?? presetPayload(DECISION_PRESETS[0], endpoint, decisionModels[0]); + const draftModel = drafts[endpoint] === undefined ? decisionModels[0] : payloadModel(rawPayload); + const activePreset = matchPreset(endpoint, rawPayload); + const clearedPayload = emptyPayload(endpoint, draftModel); const activeController = useRef(null); const validated = useMemo( () => @@ -95,9 +124,11 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = } } - function handleResetExample() { - clearRequestState(); - setDrafts((current) => Object.fromEntries(Object.entries(current).filter(([key]) => key !== endpoint))); + function handlePresetPick(id: string | null) { + const preset = DECISION_PRESETS.find((candidate) => candidate.id === id); + if (preset !== undefined) { + handlePayloadChange(presetPayload(preset, endpoint, draftModel)); + } } function handleModelPick(model: string | null) { @@ -204,9 +235,14 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = )}
- {view === "json" && (
+ {litellmParams.guardrail === "decision_model" && litellmParams.decision_model && ( + + )} {piiEntityCount > 0 && (

PII Protection

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx index 05bec9208f7..aa1d501294e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx @@ -10,6 +10,7 @@ vi.mock("@/components/networking", () => ({ getGuardrailProviderSpecificParams: vi.fn(), getGuardrailUISettings: vi.fn(), modelAvailableCall: vi.fn(), + modelHubCall: vi.fn(), })); import * as networking from "@/components/networking"; @@ -63,6 +64,7 @@ describe("AddGuardrailForm create payload characterization", () => { vi.mocked(networking.getGuardrailUISettings).mockResolvedValue(uiSettings); vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue(providerParams); vi.mocked(networking.modelAvailableCall).mockResolvedValue({ data: [{ id: "gpt-5" }] }); + vi.mocked(networking.modelHubCall).mockResolvedValue({ data: [] }); vi.mocked(networking.createGuardrailCall).mockResolvedValue({ guardrail_id: "new" }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.decision_model.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.decision_model.integration.test.tsx new file mode 100644 index 00000000000..2cdc39ea856 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.decision_model.integration.test.tsx @@ -0,0 +1,540 @@ +import React from "react"; +import { fireEvent, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "@/../tests/test-utils"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + createGuardrailCall, + decisionsTestCall, + getGuardrailProviderSpecificParams, + getGuardrailUISettings, + modelHubCall, +} from "@/components/networking"; +import { toast } from "@/lib/toast"; +import AddGuardrailForm from "./add_guardrail_form"; +import { GUARDRAIL_PRESETS } from "./guardrail_garden_configs"; + +vi.mock("@/lib/toast", () => ({ + toast: { + success: vi.fn(), + info: vi.fn(), + warning: vi.fn(), + error: vi.fn(), + fromError: vi.fn(), + dismiss: vi.fn(), + }, +})); + +vi.mock("@/components/networking", () => ({ + createGuardrailCall: vi.fn(), + decisionsTestCall: vi.fn(), + getGuardrailProviderSpecificParams: vi.fn().mockResolvedValue({}), + getGuardrailUISettings: vi.fn().mockResolvedValue({}), + modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }), + modelHubCall: vi.fn().mockResolvedValue({ data: [] }), +})); + +describe("AddGuardrailForm decision model questions", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.unstubAllGlobals(); + vi.mocked(getGuardrailProviderSpecificParams).mockResolvedValue({ + decision_model: { ui_friendly_name: "Decision Model" }, + }); + vi.mocked(getGuardrailUISettings).mockResolvedValue({ + decision_model_providers: ["typesafe", "openai"], + }); + vi.mocked(modelHubCall).mockResolvedValue({ + data: [ + { model_group: "jev-latest", providers: ["typesafe"], mode: "evaluation" }, + { model_group: "gpt-5", providers: ["openai"], mode: "evaluation" }, + ], + }); + }); + + const renderDecisionModel = async (user: ReturnType) => { + renderWithProviders( + , + ); + fireEvent.change(await screen.findByLabelText("Guardrail Name"), { target: { value: "dm-1" } }); + await user.click(screen.getByLabelText("Guardrail Provider")); + await user.click(await screen.findByText("Decision Model")); + await user.click(screen.getByRole("button", { name: "Next" })); + }; + + const pickTypesafeModel = async (user: ReturnType) => { + await user.click(await screen.findByLabelText("Decision Model")); + await user.click(await screen.findByTitle("jev-latest")); + }; + + const addQuestion = async (name: string, instructions: string) => { + fireEvent.click(await screen.findByRole("button", { name: "Add question" })); + const nameInputs = screen.getAllByLabelText(/^Question \d+ name$/); + fireEvent.change(nameInputs[nameInputs.length - 1], { target: { value: name } }); + fireEvent.change(screen.getByLabelText(`Question ${nameInputs.length}`), { target: { value: instructions } }); + }; + + it("lists every decision model on the proxy with its provider logo and no provider picker", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + vi.mocked(modelHubCall).mockResolvedValue({ + data: [ + { model_group: "jev-latest", providers: ["typesafe"], mode: "evaluation" }, + { model_group: "gpt-5", providers: ["openai"], mode: "evaluation" }, + { model_group: "gpt-4o-mini", providers: ["openai"], mode: "chat" }, + ], + }); + await renderDecisionModel(user); + + await user.click(await screen.findByLabelText("Decision Model")); + const jev = await screen.findByTitle("jev-latest"); + expect(within(jev).getByRole("img", { name: "TypeSafe logo" })).toBeInTheDocument(); + expect(within(screen.getByTitle("gpt-5")).getByRole("img", { name: "OpenAI logo" })).toBeInTheDocument(); + expect(screen.queryByTitle("gpt-4o-mini")).not.toBeInTheDocument(); + expect(screen.queryByLabelText("Decision Provider")).not.toBeInTheDocument(); + expect(screen.queryByRole("note", { name: "No decision models" })).not.toBeInTheDocument(); + + await user.click(jev); + expect(screen.getByLabelText("Decision Model")).toHaveValue("jev-latest"); + }); + + it("prompts to add a model when the proxy has no decision models", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + vi.mocked(modelHubCall).mockResolvedValue({ + data: [{ model_group: "gpt-6-luna", providers: ["openai"], mode: "chat" }], + }); + await renderDecisionModel(user); + + const note = await screen.findByRole("note", { name: "No decision models" }); + expect(note).toHaveTextContent("Add a decision model to use this guardrail"); + expect(within(note).getByRole("link", { name: "Add a model" })).toHaveAttribute( + "href", + expect.stringMatching(/\/models-and-endpoints$/), + ); + expect(screen.getByLabelText("Decision Model")).toHaveAttribute("placeholder", "No decision models yet"); + }); + + const renderFromCard = (visible: boolean, accessToken = "test-token") => ( + + ); + + it("prefills the garden card's provider model and lists only that provider's models", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + vi.mocked(modelHubCall).mockResolvedValue({ + data: [ + { model_group: "jev-preview", providers: ["typesafe"], mode: "evaluation" }, + { model_group: "gpt-5", providers: ["openai"], mode: "evaluation" }, + { model_group: "jev-latest", providers: ["typesafe"], mode: "evaluation" }, + ], + }); + renderWithProviders(renderFromCard(true)); + await waitFor(() => expect(screen.getByLabelText("Guardrail Name")).toHaveValue("TypeSafe Jev")); + await user.click(screen.getByRole("button", { name: "Next" })); + + expect(await screen.findByLabelText("Decision Model")).toHaveValue("jev-latest"); + await user.click(screen.getByLabelText("Decision Model")); + expect(await screen.findByTitle("jev-preview")).toBeInTheDocument(); + expect(screen.queryByTitle("gpt-5")).not.toBeInTheDocument(); + }); + + it("prefills the card's provider model again after the form is closed and reopened", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + const { rerender } = renderWithProviders(renderFromCard(true)); + await waitFor(() => expect(screen.getByLabelText("Guardrail Name")).toHaveValue("TypeSafe Jev")); + await user.click(screen.getByRole("button", { name: "Next" })); + expect(await screen.findByLabelText("Decision Model")).toHaveValue("jev-latest"); + + await user.click(screen.getByRole("button", { name: "Cancel" })); + rerender(renderFromCard(false)); + rerender(renderFromCard(true)); + await waitFor(() => expect(screen.getByLabelText("Guardrail Name")).toHaveValue("TypeSafe Jev")); + await user.click(screen.getByRole("button", { name: "Next" })); + + expect(await screen.findByLabelText("Decision Model")).toHaveValue("jev-latest"); + }); + + it("keeps a model the user picked when the model list reloads", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + vi.mocked(modelHubCall).mockResolvedValue({ + data: [ + { model_group: "jev-latest", providers: ["typesafe"], mode: "evaluation" }, + { model_group: "jev-preview", providers: ["typesafe"], mode: "evaluation" }, + ], + }); + const { rerender } = renderWithProviders(renderFromCard(true)); + await waitFor(() => expect(screen.getByLabelText("Guardrail Name")).toHaveValue("TypeSafe Jev")); + await user.click(screen.getByRole("button", { name: "Next" })); + expect(await screen.findByLabelText("Decision Model")).toHaveValue("jev-latest"); + await user.click(screen.getByLabelText("Decision Model")); + await user.click(await screen.findByTitle("jev-preview")); + + vi.mocked(modelHubCall).mockResolvedValue({ + data: [ + { model_group: "jev-1.13.0", providers: ["typesafe"], mode: "evaluation" }, + { model_group: "jev-latest", providers: ["typesafe"], mode: "evaluation" }, + { model_group: "jev-preview", providers: ["typesafe"], mode: "evaluation" }, + ], + }); + rerender(renderFromCard(true, "refreshed-token")); + await user.click(screen.getByLabelText("Decision Model")); + expect(await screen.findByTitle("jev-1.13.0")).toBeInTheDocument(); + expect(screen.getByLabelText("Decision Model")).toHaveValue("jev-preview"); + }); + + it("leaves the model empty when the form is not opened from a card", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + await renderDecisionModel(user); + + expect(await screen.findByLabelText("Decision Model")).toHaveValue(""); + }); + + it("names the card's provider when it has no decision models", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + renderWithProviders( + , + ); + await waitFor(() => expect(screen.getByLabelText("Guardrail Name")).toHaveValue("Databricks Decision Model")); + await user.click(screen.getByRole("button", { name: "Next" })); + + expect(await screen.findByRole("note", { name: "No decision models" })).toHaveTextContent( + "Add a decision model from the Databricks (Qwen API) provider to use this guardrail", + ); + }); + + it("shows only an Add question button until it is clicked, then a blank question per click", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + await renderDecisionModel(user); + await pickTypesafeModel(user); + + expect(screen.queryByLabelText(/^Question \d+ name$/)).not.toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: "Add question" })); + expect(screen.getByLabelText("Question 1 name")).toHaveValue(""); + expect(screen.getByLabelText("Question 1")).toHaveValue(""); + expect(screen.getByText("0.70")).toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: "Add question" })); + expect(screen.getByLabelText("Question 2 name")).toHaveValue(""); + }); + + it("submits a question with the chosen action and the default threshold", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + await renderDecisionModel(user); + await pickTypesafeModel(user); + await addQuestion("invoice_policy", "Does the text ask about invoices?"); + + await user.click(screen.getByLabelText("Question 1 action")); + await user.click(await screen.findByText("Log only")); + + fireEvent.click(screen.getByRole("button", { name: "Create Guardrail" })); + + await waitFor(() => expect(vi.mocked(createGuardrailCall)).toHaveBeenCalled()); + const submitted = vi.mocked(createGuardrailCall).mock.calls[0][1]; + expect(submitted.litellm_params.checks).toEqual([ + { + name: "invoice_policy", + instructions: "Does the text ask about invoices?", + action: "log", + threshold: 0.7, + }, + ]); + }); + + it("submits a question edited after another was added", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + await renderDecisionModel(user); + await pickTypesafeModel(user); + await addQuestion("invoice_policy", "Does the text ask about invoices?"); + await addQuestion("refund_policy", "Is this about refunds?"); + + fireEvent.change(screen.getByLabelText("Question 1 name"), { target: { value: "billing_policy" } }); + fireEvent.change(screen.getByLabelText("Question 1"), { target: { value: "Is this about billing?" } }); + fireEvent.click(screen.getByRole("button", { name: "Create Guardrail" })); + + await waitFor(() => expect(vi.mocked(createGuardrailCall)).toHaveBeenCalled()); + const submitted = vi.mocked(createGuardrailCall).mock.calls[0][1]; + expect( + submitted.litellm_params.checks.map((c: { name: string; instructions: string }) => [c.name, c.instructions]), + ).toEqual([ + ["billing_policy", "Is this about billing?"], + ["refund_policy", "Is this about refunds?"], + ]); + }); + + it("flags two questions with the same name and blocks create", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + await renderDecisionModel(user); + await pickTypesafeModel(user); + await addQuestion("invoice_policy", "Does the text ask about invoices?"); + await addQuestion("invoice_policy", "Duplicate name"); + + expect(screen.getAllByText("Another question already uses this name")).toHaveLength(2); + + fireEvent.click(screen.getByRole("button", { name: "Create Guardrail" })); + await waitFor(() => + expect(vi.mocked(toast.fromError)).toHaveBeenCalledWith("Each question needs a different name"), + ); + expect(vi.mocked(createGuardrailCall)).not.toHaveBeenCalled(); + }); + + it("blocks create while a question has no text", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + await renderDecisionModel(user); + await pickTypesafeModel(user); + await addQuestion("invoice_policy", "Does the text ask about invoices?"); + fireEvent.click(screen.getByRole("button", { name: "Add question" })); + fireEvent.change(screen.getByLabelText("Question 2 name"), { target: { value: "refund_policy" } }); + + fireEvent.click(screen.getByRole("button", { name: "Create Guardrail" })); + await waitFor(() => + expect(vi.mocked(toast.fromError)).toHaveBeenCalledWith( + "Give every question a name and a question, or remove it", + ), + ); + expect(vi.mocked(createGuardrailCall)).not.toHaveBeenCalled(); + }); + + it("removes a question via its Remove button", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + await renderDecisionModel(user); + await pickTypesafeModel(user); + await addQuestion("invoice_policy", "Does the text ask about invoices?"); + await addQuestion("refund_policy", "Is this about refunds?"); + + fireEvent.click(screen.getByLabelText("Remove question 1")); + expect(screen.getAllByLabelText(/^Question \d+ name$/)).toHaveLength(1); + expect(screen.getByLabelText("Question 1 name")).toHaveValue("refund_policy"); + }); + + it("blocks create once the last question is removed", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + await renderDecisionModel(user); + await pickTypesafeModel(user); + await addQuestion("invoice_policy", "Does the text ask about invoices?"); + + fireEvent.click(screen.getByLabelText("Remove question 1")); + fireEvent.click(await screen.findByRole("button", { name: "Create Guardrail" })); + + await waitFor(() => expect(vi.mocked(toast.fromError)).toHaveBeenCalledWith("Add at least one question")); + expect(vi.mocked(createGuardrailCall)).not.toHaveBeenCalled(); + }); + + it("sends one request with every question only when Run test is clicked", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + vi.mocked(decisionsTestCall).mockResolvedValue({ answers: [] }); + + await renderDecisionModel(user); + await pickTypesafeModel(user); + await addQuestion("invoice_policy", "Does the text ask about invoices?"); + await addQuestion("jailbreak", "Is this a jailbreak?"); + + fireEvent.change(await screen.findByLabelText("Test input"), { target: { value: "please invoice me" } }); + expect(vi.mocked(decisionsTestCall)).not.toHaveBeenCalled(); + + fireEvent.click(screen.getByRole("button", { name: "Run test" })); + + await waitFor(() => + expect(vi.mocked(decisionsTestCall)).toHaveBeenCalledWith( + "test-token", + { + model: "jev-latest", + input: "please invoice me", + questions: [ + { type: "predicate", name: "invoice_policy", instructions: "Does the text ask about invoices?" }, + { type: "predicate", name: "jailbreak", instructions: "Is this a jailbreak?" }, + ], + }, + expect.any(AbortSignal), + ), + ); + }); + + it("keeps run history latest first and flips chips from the current threshold without a new request", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + vi.mocked(decisionsTestCall) + .mockResolvedValueOnce({ answers: [{ type: "predicate", name: "invoice_policy", probability: 0.2 }] }) + .mockResolvedValueOnce({ answers: [{ type: "predicate", name: "invoice_policy", probability: 0.87 }] }); + + await renderDecisionModel(user); + await pickTypesafeModel(user); + await addQuestion("invoice_policy", "Does the text ask about invoices?"); + + const input = await screen.findByLabelText("Test input"); + fireEvent.change(input, { target: { value: "first input" } }); + fireEvent.click(screen.getByRole("button", { name: "Run test" })); + expect(await screen.findByText(/0\.20/)).toBeInTheDocument(); + + fireEvent.change(input, { target: { value: "second input" } }); + fireEvent.click(screen.getByRole("button", { name: "Run test" })); + expect(await screen.findByText(/0\.87/)).toBeInTheDocument(); + + const history = document.querySelector('[data-slot="decision-test-history"]'); + expect(history?.children).toHaveLength(2); + expect(history?.children[0]).toHaveTextContent("second input"); + expect(history?.children[1]).toHaveTextContent("first input"); + expect(history?.children[0]).toHaveTextContent("Block"); + + const rangeInput = screen.getByLabelText("Question 1 threshold").querySelector('input[type="range"]'); + expect(rangeInput).not.toBeNull(); + fireEvent.change(rangeInput!, { target: { value: "1" } }); + + expect(history?.children[0]).toHaveTextContent("Pass"); + expect(vi.mocked(decisionsTestCall)).toHaveBeenCalledTimes(2); + }); + + it("shows No answer for a refused question", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + vi.mocked(decisionsTestCall).mockResolvedValue({ answers: [{ type: "refusal", name: "invoice_policy" }] }); + + await renderDecisionModel(user); + await pickTypesafeModel(user); + await addQuestion("invoice_policy", "Does the text ask about invoices?"); + + fireEvent.change(await screen.findByLabelText("Test input"), { target: { value: "x" } }); + fireEvent.click(screen.getByRole("button", { name: "Run test" })); + + expect(await screen.findByText(/No answer/)).toBeInTheDocument(); + }); +}); + +describe("AddGuardrailForm provider search", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("finds Decision Model by search aliases and keeps label search working", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + vi.mocked(getGuardrailProviderSpecificParams).mockResolvedValue({ + decision_model: { ui_friendly_name: "Decision Model" }, + }); + renderWithProviders( + , + ); + + const provider = await screen.findByLabelText("Guardrail Provider"); + await user.click(provider); + + await user.type(provider, "jev"); + expect(await screen.findByText("Decision Model")).toBeInTheDocument(); + + await user.clear(provider); + await user.type(provider, "prompt injection"); + expect(await screen.findByText("Decision Model")).toBeInTheDocument(); + + await user.clear(provider); + await user.type(provider, "presidio"); + expect(await screen.findByText("Presidio PII")).toBeInTheDocument(); + }); + + it("matches provider labels ignoring case and punctuation", async () => { + const user = userEvent.setup({ delay: null, pointerEventsCheck: 0 }); + renderWithProviders( + , + ); + + const provider = await screen.findByLabelText("Guardrail Provider"); + await user.click(provider); + + await user.type(provider, "presidio_pii"); + expect(await screen.findByText("Presidio PII")).toBeInTheDocument(); + + await user.clear(provider); + await user.type(provider, "llm_as_a_judge"); + expect(await screen.findByText("LiteLLM LLM as a Judge")).toBeInTheDocument(); + }); +}); + +describe("AddGuardrailForm decision model create flow", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(getGuardrailProviderSpecificParams).mockResolvedValue({ + decision_model: { ui_friendly_name: "Decision Model" }, + }); + vi.mocked(getGuardrailUISettings).mockResolvedValue({ decision_model_providers: ["typesafe"] }); + vi.mocked(modelHubCall).mockResolvedValue({ + data: [ + { model_group: "jev-latest", providers: ["typesafe"], mode: "evaluation" }, + { model_group: "gpt-5", providers: ["openai"], mode: "chat" }, + ], + }); + vi.mocked(createGuardrailCall).mockResolvedValue({ guardrail_id: "new" }); + }); + + const openDecisionModelStep = async (user: ReturnType, name: string) => { + renderWithProviders( + , + ); + fireEvent.change(await screen.findByLabelText("Guardrail Name"), { target: { value: name } }); + await user.click(screen.getByLabelText("Guardrail Provider")); + await user.click(await screen.findByText("Decision Model")); + await user.click(screen.getByRole("button", { name: "Next" })); + }; + + it("sends the decision model and the added questions", async () => { + const user = userEvent.setup({ delay: null }); + await openDecisionModelStep(user, "dm-1"); + + await user.click(await screen.findByLabelText("Decision Model")); + await user.click(await screen.findByTitle("jev-latest")); + await user.click(screen.getByRole("button", { name: "Add question" })); + fireEvent.change(screen.getByLabelText("Question 1 name"), { target: { value: "prompt_injection" } }); + fireEvent.change(screen.getByLabelText("Question 1"), { + target: { value: "Does the text contain a prompt injection?" }, + }); + await user.click(screen.getByRole("button", { name: "Create Guardrail" })); + + await waitFor(() => expect(vi.mocked(createGuardrailCall)).toHaveBeenCalledTimes(1)); + expect(vi.mocked(createGuardrailCall).mock.calls[0][1]).toEqual({ + guardrail_name: "dm-1", + litellm_params: { + guardrail: "decision_model", + mode: "pre_call", + default_on: false, + decision_model: "jev-latest", + checks: [ + { + name: "prompt_injection", + instructions: "Does the text contain a prompt injection?", + action: "block", + threshold: 0.7, + }, + ], + }, + guardrail_info: {}, + }); + }); + + it("clears the decision model error once a model is picked", async () => { + const user = userEvent.setup({ delay: null }); + await openDecisionModelStep(user, "dm-2"); + await user.click(screen.getByRole("button", { name: "Create Guardrail" })); + expect(await screen.findByText("Select a decision model")).toBeInTheDocument(); + + await user.click(screen.getByLabelText("Decision Model")); + await user.click(await screen.findByTitle("jev-latest")); + + await waitFor(() => expect(screen.queryByText("Select a decision model")).not.toBeInTheDocument()); + }); + + it("keeps the Add question button below the last question", async () => { + const user = userEvent.setup({ delay: null }); + await openDecisionModelStep(user, "dm-3"); + await user.click(await screen.findByRole("button", { name: "Add question" })); + await user.click(screen.getByRole("button", { name: "Add question" })); + + const lastQuestion = screen.getByLabelText("Question 2"); + const addButton = screen.getByRole("button", { name: "Add question" }); + expect(lastQuestion.compareDocumentPosition(addButton) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx index 88093ac4517..b1d5c6d8929 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx @@ -6,6 +6,7 @@ import { getGuardrailProviderSpecificParams, getGuardrailUISettings, modelAvailableCall, + modelHubCall, } from "@/components/networking"; import ContentFilterConfiguration from "./content_filter/ContentFilterConfiguration"; import { type CompetitorIntentConfig } from "./content_filter/CompetitorIntentConfiguration"; @@ -17,11 +18,13 @@ import { getGuardrailProviders, getSupportedModesForProvider, guardrail_provider_map, + guardrail_provider_search_aliases, effectiveLoggingOnlyContinue, modeIncludesLoggingOnly, populateGuardrailProviderMap, populateGuardrailProviders, shouldRenderContentFilterConfigSettings, + shouldRenderDecisionModelFields, shouldRenderLLMJudgeFields, shouldRenderPIIConfigSettings, streamScopePayload, @@ -43,6 +46,7 @@ import { ComboboxInput, ComboboxItem, ComboboxList, + useComboboxFilter, } from "@/components/ui/combobox"; import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; @@ -64,6 +68,17 @@ import { import GuardrailOptionalParams from "./guardrail_optional_params"; import GuardrailProviderFields from "./guardrail_provider_fields"; import LLMJudgeFields from "./llm_judge/LLMJudgeFields"; +import DecisionModelFields from "./decision_model/DecisionModelFields"; +import { + buildDecisionModelParams, + decisionChecksProblem, + type DecisionModelCheckDraft, +} from "./decision_model/buildDecisionModelParams"; +import { + decisionModelOptions, + parseDecisionModelGroups, + type DecisionModelGroup, +} from "./decision_model/decisionModelQuestion"; import PiiConfiguration from "./pii_configuration"; import ToolPermissionRulesEditor, { ToolPermissionConfig } from "./tool_permission/ToolPermissionRulesEditor"; import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog"; @@ -87,6 +102,7 @@ interface GuardrailPreset { // modes seeded, not one; the form already normalises either shape. mode: string | string[]; defaultOn: boolean; + decisionProvider?: string; } interface AddGuardrailFormProps { @@ -107,6 +123,7 @@ interface GuardrailSettings { category: string; entities: string[]; }>; + decision_model_providers?: string[]; content_filter_settings?: { prebuilt_patterns: Array<{ name: string; @@ -218,6 +235,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a const watchedMode = useWatch({ control: form.control, name: "mode" }); const [loading, setLoading] = useState(false); const [selectedProvider, setSelectedProvider] = useState(null); + const providerFilter = useComboboxFilter(); const [guardrailSettings, setGuardrailSettings] = useState(null); const [selectedEntities, setSelectedEntities] = useState([]); const [selectedActions, setSelectedActions] = useState<{ [key: string]: string }>({}); @@ -244,6 +262,10 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a createEmptyToolPermissionConfig, ); + const [decisionModelGroups, setDecisionModelGroups] = useState([]); + const [decisionProvider, setDecisionProvider] = useState(null); + const [decisionChecks, setDecisionChecks] = useState([]); + const isToolPermissionProvider = useMemo(() => { if (!selectedProvider) { return false; @@ -251,6 +273,17 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a const providerValue = guardrail_provider_map[selectedProvider]; return (providerValue || "").toLowerCase() === "tool_permission"; }, [selectedProvider]); + const isDecisionModelProvider = useMemo(() => shouldRenderDecisionModelFields(selectedProvider), [selectedProvider]); + const decisionModels = useMemo( + () => + decisionModelOptions(guardrailSettings?.decision_model_providers ?? [], decisionModelGroups, decisionProvider), + [guardrailSettings, decisionModelGroups, decisionProvider], + ); + const revalidateDecisionModel = () => { + if (form.getFieldState("decision_model").error) { + void form.trigger("decision_model"); + } + }; const directionalScopeSupported = supportsDirectionalLoggingOnlyScope(guardrailSettings, selectedProvider); // Fetch guardrail UI settings + provider params on mount / accessToken change @@ -260,10 +293,11 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a const fetchData = async () => { try { // Parallel requests for speed - const [uiSettings, providerParamsResp, modelsResp] = await Promise.all([ + const [uiSettings, providerParamsResp, modelsResp, modelGroupsResp] = await Promise.all([ getGuardrailUISettings(accessToken), getGuardrailProviderSpecificParams(accessToken), modelAvailableCall(accessToken, "", "").catch(() => null), + modelHubCall(accessToken).catch(() => null), ]); setGuardrailSettings(uiSettings); @@ -275,6 +309,8 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a // Populate dynamic providers from API response populateGuardrailProviders(providerParamsResp); populateGuardrailProviderMap(providerParamsResp); + + setDecisionModelGroups(parseDecisionModelGroups(modelGroupsResp?.data)); } catch (error) { console.error("Error fetching guardrail data:", error); toast.fromError("Failed to load guardrail configuration"); @@ -290,6 +326,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a // Set provider setSelectedProvider(preset.provider); + setDecisionProvider(preset.decisionProvider ?? null); const baseValues: Record = { provider: preset.provider, guardrail_name: preset.guardrailNameSuggestion, @@ -324,6 +361,12 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a } }, [preset, visible, guardrailSettings, form]); + useEffect(() => { + if (!visible || !decisionProvider || form.getValues("decision_model")) return; + const firstModel = decisionModels[0]?.model; + if (firstModel) form.setValue("decision_model", firstModel); + }, [visible, decisionProvider, decisionModels, form]); + const handleProviderChange = (value: string) => { setSelectedProvider(value); // Reset form fields that are provider-specific @@ -370,6 +413,9 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a if (value === "LlmAsAJudge") { form.setValue("mode", "post_call"); } + + setDecisionChecks([]); + setDecisionProvider(null); }; const handleEntitySelect = (entity: string) => { @@ -406,6 +452,16 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a toast.fromError("Please select at least one PII entity to continue"); return; } + if (isDecisionModelProvider) { + if (!(await form.trigger(["decision_model"]))) { + return; + } + const questionsProblem = decisionChecksProblem(decisionChecks); + if (questionsProblem) { + toast.fromError(questionsProblem); + return; + } + } } setCurrentStep(currentStep + 1); @@ -425,6 +481,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a setSelectedContentCategories([]); setPendingCategorySelection(""); setToolPermissionConfig(createEmptyToolPermissionConfig()); + setDecisionChecks([]); setSelectedEndpointType(""); setEndSessionAfterNFails(undefined); setOnViolation("warn"); @@ -617,6 +674,18 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a })); } + if (guardrailProvider === "decision_model") { + const questionsProblem = decisionChecksProblem(decisionChecks); + if (questionsProblem) { + toast.fromError(questionsProblem); + setLoading(false); + return; + } + const decisionParams = buildDecisionModelParams(asText(values.decision_model), decisionChecks); + guardrailData.litellm_params.decision_model = decisionParams.decision_model; + guardrailData.litellm_params.checks = decisionParams.checks; + } + if (guardrailProvider === "tool_permission") { if (toolPermissionConfig.rules.length === 0) { toast.fromError("Add at least one tool permission rule"); @@ -654,8 +723,9 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a ******************************/ // Use pre-fetched provider params to copy recognised params - // Skip for providers that handle their own litellm_params (llm_as_a_judge, tool_permission, content filter, PII) - if (providerParams && selectedProvider && guardrailProvider !== "llm_as_a_judge") { + // Skip for providers that handle their own litellm_params (llm_as_a_judge, tool_permission, content filter, PII, decision_model) + const providerOwnsLitellmParams = ["llm_as_a_judge", "decision_model"].includes(guardrailProvider); + if (providerParams && selectedProvider && !providerOwnsLitellmParams) { const providerKey = guardrail_provider_map[selectedProvider]?.toLowerCase(); const providerSpecificParams = providerParams[providerKey] || {}; @@ -710,12 +780,19 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a }; const renderBasicInfo = () => { - const showProviderFields = - !isToolPermissionProvider && - !shouldRenderContentFilterConfigSettings(selectedProvider) && - !shouldRenderLLMJudgeFields(selectedProvider); + const usesOwnConfigurationFields = [ + isToolPermissionProvider, + shouldRenderContentFilterConfigSettings(selectedProvider), + shouldRenderLLMJudgeFields(selectedProvider), + isDecisionModelProvider, + ].some(Boolean); + const showProviderFields = !usesOwnConfigurationFields; const providerLabels: Record = getGuardrailProviders(); const providerKeys = Object.keys(providerLabels); + const providerMatchesQuery = (key: string, query: string) => + [providerLabels[key] ?? key, ...(guardrail_provider_search_aliases[guardrail_provider_map[key]] ?? [])].some( + (term) => providerFilter.contains(term, query), + ); const supportedModes = getSupportedModesForProvider(guardrailSettings, selectedProvider) ?? DEFAULT_MODES; return ( @@ -740,6 +817,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a providerLabels[key] ?? key} + filter={providerMatchesQuery} value={asText(value) || null} onValueChange={(key: string | null) => { onChange(key ?? ""); @@ -964,6 +1042,19 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a if (shouldRenderLLMJudgeFields(selectedProvider)) { return ; } + if (isDecisionModelProvider) { + return ( + + ); + } return renderOptionalParams(); case 2: if (shouldRenderContentFilterConfigSettings(selectedProvider)) { @@ -1135,6 +1226,12 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a { title: "PII Configuration", optional: false }, ]; } + if (isDecisionModelProvider) { + return [ + { title: "Basic Info", optional: false }, + { title: "Decision Model Configuration", optional: false }, + ]; + } return [ { title: "Basic Info", optional: false }, { title: "Provider Configuration", optional: false }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/decision_model/DecisionModelFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/decision_model/DecisionModelFields.tsx new file mode 100644 index 00000000000..34e721bfae5 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/decision_model/DecisionModelFields.tsx @@ -0,0 +1,253 @@ +"use client"; + +import { Info, Plus, X } from "lucide-react"; +import React from "react"; +import { useWatch } from "react-hook-form"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; +import { Button } from "@/components/ui/button"; +import { + Combobox, + ComboboxContent, + ComboboxEmpty, + ComboboxInput, + ComboboxItem, + ComboboxList, +} from "@/components/ui/combobox"; +import { Field, FieldLabel } from "@/components/ui/field"; +import { Input } from "@/components/ui/input"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Slider } from "@/components/ui/slider"; +import { Textarea } from "@/components/ui/textarea"; +import { Logo } from "@/components/molecules/logo/Logo"; +import { getProviderLogoAndName } from "@/components/provider_info_helpers"; +import { uiHref } from "@/utils/uiHref"; +import { asText, GuardrailField, labelWithHint, requiredRule, type GuardrailFormControl } from "../GuardrailFormField"; +import { duplicateDecisionCheckNames, type DecisionModelCheckDraft } from "./buildDecisionModelParams"; +import DecisionTestSection from "./DecisionTestSection"; +import { newDecisionCheckDraft, type DecisionModelOption } from "./decisionModelQuestion"; + +export interface DecisionModelFieldsProps { + accessToken: string | null; + providerFilter: string | null; + onModelChange: () => void; + decisionModels: DecisionModelOption[]; + checks: DecisionModelCheckDraft[]; + onChecksChange: (checks: DecisionModelCheckDraft[]) => void; + control: GuardrailFormControl; +} + +const ACTION_ITEMS = [ + { label: "Block", value: "block" }, + { label: "Log only", value: "log" }, +]; + +const ProviderLogo: React.FC<{ provider: string }> = ({ provider }) => ( + +); + +const NoDecisionModels: React.FC<{ providerFilter: string | null }> = ({ providerFilter }) => { + const source = providerFilter ? ` from the ${getProviderLogoAndName(providerFilter).displayName} provider` : ""; + return ( + + + No decision models yet + +

+ Add a decision model{source} to use this guardrail. A model shows up here once its mode is evaluation.{" "} + + Add a model + +

+ + + ); +}; + +const ThresholdSlider: React.FC<{ + value: number; + onChange: (value: number) => void; + "aria-label": string; +}> = ({ value, onChange, "aria-label": ariaLabel }) => ( +
+ onChange(Array.isArray(next) ? next[0] ?? 0 : next)} + className="min-w-[120px] flex-1 **:data-[slot=slider-track]:bg-muted-foreground/25" + /> + + {value.toFixed(2)} + +
+); + +const DecisionModelFields: React.FC = ({ + accessToken, + providerFilter, + onModelChange, + decisionModels, + checks, + onChecksChange, + control, +}) => { + const selectedModel = asText(useWatch({ control, name: "decision_model" })); + const duplicateNames = duplicateDecisionCheckNames(checks); + const providerByModel = new Map(decisionModels.map((option) => [option.model, option.provider])); + + const updateCheck = (id: string, patch: Partial) => + onChecksChange(checks.map((check) => (check.id === id ? { ...check, ...patch } : check))); + + const addCheck = () => onChecksChange([...checks, newDecisionCheckDraft(checks)]); + + const removeCheck = (id: string) => onChecksChange(checks.filter((check) => check.id !== id)); + + return ( +
+
+ The Decision Model answers yes/no questions about each request (pre_call, during_call) or + response (post_call). A question whose probability reaches its threshold blocks the call, or is recorded when + set to Log only. +
+ + + {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( + option.model)} + value={asText(value) || null} + onValueChange={(next: string | null) => { + onChange(next); + onModelChange(); + }} + > + 0 ? "Select a decision model" : "No decision models yet"} + className="w-full" + /> + + No matching models + + {(model: string) => { + const provider = providerByModel.get(model); + return ( + + + {provider && } + {model} + + + ); + }} + + + + )} + + + {decisionModels.length === 0 && } + + + Questions +

+ Yes/no questions the decision model answers about the text. Block stops the call once a score reaches the + threshold, and Log only records it. +

+ {checks.map((check, index) => { + const position = index + 1; + return ( +
+ +
+ Name + updateCheck(check.id, { name: event.target.value })} + /> + {duplicateNames.has(check.name.trim()) && ( +

Another question already uses this name

+ )} +
+
+ Question +